utils.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616
  1. /**
  2. * @license Copyright (c) 2003-2016, CKSource - Frederico Knabben. All rights reserved.
  3. * For licensing, see LICENSE.md.
  4. */
  5. 'use strict';
  6. const path = require( 'path' );
  7. const gulp = require( 'gulp' );
  8. const rename = require( 'gulp-rename' );
  9. const replace = require( 'gulp-replace' );
  10. const gulpBabel = require( 'gulp-babel' );
  11. const gutil = require( 'gulp-util' );
  12. const gulpFilter = require( 'gulp-filter' );
  13. const multipipe = require( 'multipipe' );
  14. const PassThrough = require( 'stream' ).PassThrough;
  15. const through = require( 'through2' );
  16. const fs = require( 'fs' );
  17. const sass = require( 'node-sass' );
  18. const minimist = require( 'minimist' );
  19. const sprite = require( 'gulp-svg-sprite' );
  20. const pipe = require( 'multipipe' );
  21. const filter = require( 'gulp-filter' );
  22. const mainUtils = require( '../utils' );
  23. const utils = {
  24. /**
  25. * Code which can be appended to a transpiled (into AMD) test files in order to
  26. * load the 'tests' module and defer launching Bender until it's ready.
  27. *
  28. * Note: This code will not be transpiled so keep it in ES5.
  29. */
  30. benderLauncherCode:
  31. `
  32. require( [ 'tests' ], bender.defer(), function( err ) {
  33. // The problem with Require.JS is that there are no stacktraces if we won't log this.
  34. console.error( err );
  35. console.log( err.stack );
  36. } );
  37. `,
  38. /**
  39. * Module formats supported by the builder.
  40. */
  41. SUPPORTED_FORMATS: [ 'esnext', 'amd', 'cjs' ],
  42. /**
  43. * Creates a simple duplex stream.
  44. *
  45. * @param {Function} [callback] A callback which will be executed with each chunk.
  46. * @returns {Stream}
  47. */
  48. noop( callback ) {
  49. if ( !callback ) {
  50. return new PassThrough( { objectMode: true } );
  51. }
  52. return through( { objectMode: true }, ( file, encoding, throughCallback ) => {
  53. callback( file );
  54. throughCallback( null, file );
  55. } );
  56. },
  57. /**
  58. * Saves the files piped into this stream to the `build/` directory.
  59. *
  60. * @param {String} buildDir The `build/` directory path.
  61. * @param {String} format The format of the buildribution (`esnext`, `amd`, or `cjs`).
  62. * @returns {Stream}
  63. */
  64. destBuild( buildDir, format ) {
  65. const destDir = path.join( buildDir, format );
  66. return gulp.dest( destDir );
  67. },
  68. /**
  69. * Creates a function generating convertion streams.
  70. * Used to generate `formats.reduce()` callback where `formats` is an array of formats that should be generated.
  71. *
  72. * @param {String} buildDir The `build/` directory path.
  73. * @returns {Function}
  74. */
  75. getConversionStreamGenerator( buildDir ) {
  76. return ( pipes, format ) => {
  77. const conversionPipes = [];
  78. conversionPipes.push( utils.pickVersionedFile( format ) );
  79. if ( format != 'esnext' ) {
  80. // Convert src files.
  81. const filterSource = gulpFilter( ( file ) => {
  82. return utils.isSourceFile( file ) && utils.isJSFile( file );
  83. }, { restore: true } );
  84. const transpileSource = utils.transpile( format, utils.getBabelOptionsForSource( format ) );
  85. conversionPipes.push(
  86. filterSource,
  87. transpileSource,
  88. utils.fixCoverage(),
  89. filterSource.restore
  90. );
  91. // Convert test files.
  92. const filterTests = gulpFilter( ( file ) => {
  93. return utils.isTestFile( file ) && utils.isJSFile( file );
  94. }, { restore: true } );
  95. const transpileTests = utils.transpile( format, utils.getBabelOptionsForTests( format ) );
  96. conversionPipes.push(
  97. filterTests,
  98. transpileTests,
  99. format === 'amd' ? utils.appendBenderLauncher() : utils.noop(),
  100. filterTests.restore
  101. );
  102. }
  103. conversionPipes.push(
  104. utils.destBuild( buildDir, format ),
  105. utils.noop( ( file ) => {
  106. gutil.log( `Finished writing '${ gutil.colors.cyan( file.path ) }'` );
  107. } )
  108. );
  109. pipes.push( multipipe.apply( null, conversionPipes ) );
  110. return pipes;
  111. };
  112. },
  113. /**
  114. * Transpiles files piped into this stream to the given format (`amd` or `cjs`).
  115. *
  116. * @param {String} format
  117. * @returns {Stream}
  118. */
  119. transpile( format, options ) {
  120. return gulpBabel( options )
  121. .on( 'error', ( err ) => {
  122. gutil.log( gutil.colors.red( `Error (Babel:${ format })` ) );
  123. gutil.log( gutil.colors.red( err.message ) );
  124. console.log( '\n' + err.codeFrame + '\n' );
  125. } );
  126. },
  127. /**
  128. * Adds istanbul ignore to the code created by babel.
  129. *
  130. * @returns {Stream}
  131. */
  132. fixCoverage() {
  133. return replace(
  134. 'return obj && obj.__esModule ? obj :',
  135. '/* istanbul ignore next */\n\t\t' +
  136. 'return obj && obj.__esModule ? obj :' );
  137. },
  138. /**
  139. * Returns an object with Babel options for the source code.
  140. *
  141. * @param {String} format
  142. * @returns {Object} options
  143. */
  144. getBabelOptionsForSource( format ) {
  145. return {
  146. plugins: utils.getBabelPlugins( format ),
  147. // Ensure that all paths ends with '.js' because Require.JS (unlike Common.JS/System.JS)
  148. // will not add it to module names which look like paths.
  149. resolveModuleSource: format == 'cjs' ? utils.resolveModuleSource : utils.appendModuleExtension
  150. };
  151. },
  152. /**
  153. * Returns an object with Babel options for the test code.
  154. *
  155. * @param {String} format
  156. * @returns {Object} options
  157. */
  158. getBabelOptionsForTests( format ) {
  159. return {
  160. plugins: utils.getBabelPlugins( format ),
  161. resolveModuleSource: format == 'cjs' ? utils.resolveModuleSource : utils.appendModuleExtension,
  162. moduleIds: true,
  163. moduleId: 'tests'
  164. };
  165. },
  166. /**
  167. * Returns an array of Babel plugins to use.
  168. *
  169. * @param {String} format
  170. * @returns {Array}
  171. */
  172. getBabelPlugins( format ) {
  173. const babelModuleTranspilers = {
  174. amd: 'amd',
  175. cjs: 'commonjs'
  176. };
  177. const babelModuleTranspiler = babelModuleTranspilers[ format ];
  178. if ( !babelModuleTranspiler ) {
  179. throw new Error( `Incorrect format: ${ format }` );
  180. }
  181. return [
  182. // Note: When plugin is specified by its name, Babel loads it from a context of a
  183. // currently transpiled file (in our case - e.g. from ckeditor5-core/src/foo.js).
  184. // Obviously that fails, since we have all the plugins installed only in ckeditor5/
  185. // and we want to have them only there to avoid installing them dozens of times.
  186. //
  187. // Anyway, I haven't found in the docs that you can also pass a plugin instance here,
  188. // but it works... so let's hope it will.
  189. require( `babel-plugin-transform-es2015-modules-${ babelModuleTranspiler }` )
  190. ];
  191. },
  192. /**
  193. * Appends the {@link #benderLauncherCode} at the end of the file.
  194. *
  195. * @returns {Stream}
  196. */
  197. appendBenderLauncher() {
  198. return through( { objectMode: true }, ( file, encoding, callback ) => {
  199. if ( !file.isNull() ) {
  200. file.contents = new Buffer( file.contents.toString() + utils.benderLauncherCode );
  201. }
  202. callback( null, file );
  203. } );
  204. },
  205. /**
  206. * Allows us to pick one of files suffixed with the format (`__esnext`, `__amd`, or `__cjs`) and removes
  207. * files with other suffixes from the stream.
  208. *
  209. * For example: we have `load__esnext.js`, `load__amd.js` and `load__cjs.js`. After applying this
  210. * transformation when compiling code for a specific format the proper file will be renamed to `load.js`.
  211. * Files not matching a specified format will be removed.
  212. *
  213. * @param {String} format
  214. * @returns {Stream}
  215. */
  216. pickVersionedFile( format ) {
  217. const rejectedFormats = utils.SUPPORTED_FORMATS
  218. .filter( ( item ) => item !== format );
  219. const pickRegexp = new RegExp( `__${ format }$` );
  220. const rejectRegexp = new RegExp( `__(${ rejectedFormats.join( '|' ) }).js$` );
  221. const pick = rename( ( path ) => {
  222. path.basename = path.basename.replace( pickRegexp, '' );
  223. } );
  224. const remove = gulpFilter( ( file ) => !rejectRegexp.test( file.path ) );
  225. return multipipe( pick, remove );
  226. },
  227. /**
  228. * Processes paths of files inside CKEditor5 packages.
  229. *
  230. * * `ckeditor5-xxx/src/foo/bar.js` -> `ckeditor5/xxx/foo/bar.js`
  231. * * `ckeditor5-xxx/tests/foo/bar.js` -> `tests/xxx/foo/bar.js`
  232. *
  233. * @returns {Stream}
  234. */
  235. renamePackageFiles() {
  236. return rename( ( file ) => {
  237. const dirFrags = file.dirname.split( path.sep );
  238. // Validate the input for the clear conscious.
  239. if ( dirFrags[ 0 ].indexOf( 'ckeditor5-' ) !== 0 ) {
  240. throw new Error( 'Path should start with "ckeditor5-".' );
  241. }
  242. dirFrags[ 0 ] = dirFrags[ 0 ].replace( /^ckeditor5-/, '' );
  243. const firstFrag = dirFrags[ 1 ];
  244. if ( firstFrag == 'src' ) {
  245. // Remove 'src/'.
  246. dirFrags.splice( 1, 1 );
  247. // Temporary implementation of the UI lib option. See #88.
  248. if ( dirFrags[ 0 ] == 'ui-default' ) {
  249. dirFrags[ 0 ] = 'ui';
  250. }
  251. // And prepend 'ckeditor5/'.
  252. dirFrags.unshift( 'ckeditor5' );
  253. } else if ( firstFrag == 'tests' ) {
  254. // Remove 'tests/' from the package dir.
  255. dirFrags.splice( 1, 1 );
  256. // And prepend 'tests/'.
  257. dirFrags.unshift( 'tests' );
  258. } else {
  259. throw new Error( 'Path should start with "ckeditor5-*/(src|tests|theme)".' );
  260. }
  261. file.dirname = path.join.apply( null, dirFrags );
  262. } );
  263. },
  264. /**
  265. * Processes paths of files inside the main CKEditor5 package.
  266. *
  267. * * `src/foo/bar.js` -> `ckeditor5/foo/bar.js`
  268. * * `tests/foo/bar.js` -> `tests/ckeditor5/foo/bar.js`
  269. *
  270. * @returns {Stream}
  271. */
  272. renameCKEditor5Files() {
  273. return rename( ( file ) => {
  274. const dirFrags = file.dirname.split( path.sep );
  275. const firstFrag = dirFrags[ 0 ];
  276. if ( firstFrag == 'src' ) {
  277. // Replace 'src/' with 'ckeditor5/'.
  278. // src/path.js -> ckeditor5/path.js
  279. dirFrags.splice( 0, 1, 'ckeditor5' );
  280. } else if ( firstFrag == 'tests' ) {
  281. // Insert 'ckeditor5/' after 'tests/'.
  282. // tests/foo.js -> tests/ckeditor5/foo.js
  283. dirFrags.splice( 1, 0, 'ckeditor5' );
  284. } else {
  285. throw new Error( 'Path should start with "src" or "tests".' );
  286. }
  287. file.dirname = path.join.apply( null, dirFrags );
  288. } );
  289. },
  290. /**
  291. * Appends file extension to file URLs. Tries to not touch named modules.
  292. *
  293. * @param {String} source
  294. * @returns {String}
  295. */
  296. appendModuleExtension( source ) {
  297. if ( /^https?:|\.[\/\\]/.test( source ) && !/\.js$/.test( source ) ) {
  298. return source + '.js';
  299. }
  300. return source;
  301. },
  302. /**
  303. * Checks whether a file is a test file.
  304. *
  305. * @param {Vinyl} file
  306. * @returns {Boolean}
  307. */
  308. isTestFile( file ) {
  309. // TODO this should be based on bender configuration (config.tests.*.paths).
  310. if ( !file.relative.startsWith( 'tests' + path.sep ) ) {
  311. return false;
  312. }
  313. const dirFrags = file.relative.split( path.sep );
  314. return !dirFrags.some( dirFrag => {
  315. return dirFrag.startsWith( '_' ) && dirFrag != '_utils-tests';
  316. } );
  317. },
  318. /**
  319. * Checks whether a file is a source file.
  320. *
  321. * @param {Vinyl} file
  322. * @returns {Boolean}
  323. */
  324. isSourceFile( file ) {
  325. return !utils.isTestFile( file );
  326. },
  327. /**
  328. * Checks whether a file is a JS file.
  329. *
  330. * @param {Vinyl} file
  331. * @returns {Boolean}
  332. */
  333. isJSFile( file ) {
  334. return file.path.endsWith( '.js' );
  335. },
  336. /**
  337. * Finds all CKEditor5 package directories in "node_modules" folder.
  338. *
  339. * @param {String} rootDir A root directory containing "node_modules" folder.
  340. * @returns {Array} Array of ckeditor5-* package directory paths.
  341. */
  342. getPackages( rootDir ) {
  343. // Find all CKEditor5 package directories. Resolve symlinks so we watch real directories
  344. // in order to workaround https://github.com/paulmillr/chokidar/issues/419.
  345. return fs.readdirSync( path.join( rootDir, 'node_modules' ) )
  346. // Look for ckeditor5-* directories.
  347. .filter( fileName => {
  348. return fileName.indexOf( 'ckeditor5-' ) === 0;
  349. } )
  350. // Resolve symlinks and keep only directories.
  351. .map( fileName => {
  352. let filePath = path.join( rootDir, 'node_modules', fileName );
  353. let stat = fs.lstatSync( filePath );
  354. if ( stat.isSymbolicLink() ) {
  355. filePath = fs.realpathSync( filePath );
  356. stat = fs.lstatSync( filePath );
  357. }
  358. if ( stat.isDirectory() ) {
  359. return filePath;
  360. }
  361. // Filter...
  362. return false;
  363. } )
  364. // ...those out.
  365. .filter( filePath => filePath );
  366. },
  367. /**
  368. * Filters theme entry points only from a stream of SCSS files.
  369. *
  370. * @returns {Stream}
  371. */
  372. filterThemeEntryPoints() {
  373. return filter( '**/theme.scss' );
  374. },
  375. /**
  376. * Given the input stream of theme entry-point files (theme.scss), this method:
  377. * 1. Collects paths to entry-point.
  378. * 2. Builds the output CSS theme file using aggregated entry-points.
  379. * 3. Returns a stream containing built CSS theme file.
  380. *
  381. * @param {String} fileName The name of the output CSS theme file.
  382. * @returns {Stream}
  383. */
  384. compileThemes( fileName ) {
  385. const paths = [];
  386. const stream = through.obj( collectThemeEntryPoint, renderThemeFromEntryPoints );
  387. function collectThemeEntryPoint( file, enc, callback ) {
  388. paths.push( file.path );
  389. callback();
  390. }
  391. function renderThemeFromEntryPoints( callback ) {
  392. gutil.log( `Compiling '${ gutil.colors.cyan( fileName ) }' from ${ gutil.colors.cyan( paths.length ) } entry points...` );
  393. // Sort to make sure theme is the very first SASS to build. Otherwise,
  394. // packages using mixins and variables from that theme will throw errors
  395. // because such are not available at this stage of compilation.
  396. const dataToRender = paths.sort( a => -a.indexOf( 'ckeditor5-theme' ) )
  397. // Make sure windows\\style\\paths are preserved.
  398. .map( p => `@import "${ p.replace( /\\/g, '\\\\' ) }";` )
  399. .join( '\n' );
  400. try {
  401. const rendered = sass.renderSync( utils.getSassOptions( dataToRender ) );
  402. stream.push( new gutil.File( {
  403. path: fileName,
  404. contents: new Buffer( rendered.css )
  405. } ) );
  406. callback();
  407. } catch ( err ) {
  408. callback( err );
  409. }
  410. }
  411. return stream;
  412. },
  413. /**
  414. * Parses command line arguments and returns them as a user-friendly hash.
  415. *
  416. * @param {String} dataToRender
  417. * @returns {Object}
  418. */
  419. getSassOptions( dataToRender ) {
  420. return {
  421. data: dataToRender,
  422. sourceMap: true,
  423. sourceMapEmbed: true,
  424. outputStyle: 'expanded',
  425. sourceComments: true
  426. };
  427. },
  428. /**
  429. * Parses command line arguments and returns them as a user-friendly hash.
  430. *
  431. * @returns {Object} options
  432. * @returns {Array} [options.formats] Array of specified output formats ("esnext" or "amd").
  433. * @returns {Boolean} [options.watch] A flag which enables watch mode.
  434. */
  435. parseArguments() {
  436. const options = minimist( process.argv.slice( 2 ), {
  437. string: [
  438. 'formats'
  439. ],
  440. boolean: [
  441. 'watch'
  442. ],
  443. default: {
  444. formats: 'amd',
  445. watch: false
  446. }
  447. } );
  448. options.formats = options.formats.split( ',' );
  449. return options;
  450. },
  451. /**
  452. * Given a stream of .svg files it returns a stream containing JavaScript
  453. * icon sprite file.
  454. *
  455. * @returns {Stream}
  456. */
  457. compileIconSprite() {
  458. return sprite( utils.getIconSpriteOptions() );
  459. },
  460. /**
  461. * Returns svg-sprite util options to generate <symbol>-based, JavaScript
  462. * sprite file.
  463. *
  464. * @returns {Object}
  465. */
  466. getIconSpriteOptions() {
  467. return {
  468. shape: {
  469. id: {
  470. // Note: Consider unix/style/paths and windows\\style\\paths.
  471. generator: name => `ck-icon-${ name.match( /([^\/\\]*)\.svg$/ )[ 1 ] }`
  472. },
  473. },
  474. svg: {
  475. xmlDeclaration: false,
  476. doctypeDeclaration: false,
  477. },
  478. mode: {
  479. symbol: {
  480. dest: './', // Flatten symbol/ folder.
  481. inline: true,
  482. render: {
  483. js: {
  484. template: path.join( __dirname, 'iconmanagermodel.tpl' ),
  485. dest: 'iconmanagermodel.js',
  486. }
  487. }
  488. }
  489. }
  490. };
  491. },
  492. /**
  493. * Given a stream of files it returns an array of gulp-mirror streams outputting
  494. * files to `build/[formats]/theme/` directories for each of desired output formats (cjs, amd, etc.).
  495. *
  496. * @param {String} buildDir A path to /build directory.
  497. * @param {Array} formats An array of desired output formats.
  498. * @param {Function} [transformationStream] A stream used to transform files before they're saved to
  499. * desired `build/[formats]/theme` directories. Useful for transpilation.
  500. * @returns {Stream[]} An array of streams.
  501. */
  502. getThemeFormatDestStreams( buildDir, formats, transformationStream ) {
  503. return formats.map( f => {
  504. return pipe(
  505. transformationStream ? transformationStream( f ) : utils.noop(),
  506. gulp.dest( path.join( buildDir, f, 'theme' ) ),
  507. utils.noop( file => {
  508. gutil.log( `Output for ${ gutil.colors.cyan( f ) } is '${ gutil.colors.cyan( file.path ) }'.` );
  509. } )
  510. );
  511. } );
  512. },
  513. /**
  514. * Resolves CommonJS module source path.
  515. *
  516. * @param {String} source Module path passed to require() method.
  517. * @param {String} file Path to a file where require() method is called.
  518. * @returns {String} Fixed module path.
  519. */
  520. resolveModuleSource( source, file ) {
  521. // If path is relative - leave it as is.
  522. if ( !path.isAbsolute( source ) ) {
  523. return source;
  524. }
  525. // Find relative path of test file from cwd directory.
  526. let testFile = path.relative( process.cwd(), file );
  527. // Append `/` as all files uses it as root inside transpiled versions.
  528. testFile = path.join( path.sep, testFile );
  529. // Find relative path from test file to source.
  530. let relativePath = path.relative( path.dirname( testFile ), path.dirname( source ) );
  531. relativePath = path.join( relativePath, path.basename( source ) );
  532. // Convert windows path to posix.
  533. relativePath = relativePath.replace( /\\/g, '/' );
  534. return utils.appendModuleExtension( ( relativePath.startsWith( '../' ) ? '' : './' ) + relativePath );
  535. }
  536. };
  537. // Extend top level utils.
  538. module.exports = Object.assign( utils, mainUtils );