utils.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614
  1. /**
  2. * @license Copyright (c) 2003-2016, CKSource - Frederico Knabben. All rights reserved.
  3. * For licensing, see LICENSE.md.
  4. */
  5. 'use strict';
  6. const path = require( 'path' );
  7. const gulp = require( 'gulp' );
  8. const rename = require( 'gulp-rename' );
  9. const replace = require( 'gulp-replace' );
  10. const gulpBabel = require( 'gulp-babel' );
  11. const gutil = require( 'gulp-util' );
  12. const gulpFilter = require( 'gulp-filter' );
  13. const multipipe = require( 'multipipe' );
  14. const PassThrough = require( 'stream' ).PassThrough;
  15. const through = require( 'through2' );
  16. const fs = require( 'fs' );
  17. const sass = require( 'node-sass' );
  18. const minimist = require( 'minimist' );
  19. const sprite = require( 'gulp-svg-sprite' );
  20. const pipe = require( 'multipipe' );
  21. const filter = require( 'gulp-filter' );
  22. const utils = {
  23. /**
  24. * Code which can be appended to a transpiled (into AMD) test files in order to
  25. * load the 'tests' module and defer launching Bender until it's ready.
  26. *
  27. * Note: This code will not be transpiled so keep it in ES5.
  28. */
  29. benderLauncherCode:
  30. `
  31. require( [ 'tests' ], bender.defer(), function( err ) {
  32. // The problem with Require.JS is that there are no stacktraces if we won't log this.
  33. console.error( err );
  34. console.log( err.stack );
  35. } );
  36. `,
  37. /**
  38. * Module formats supported by the builder.
  39. */
  40. SUPPORTED_FORMATS: [ 'esnext', 'amd', 'cjs' ],
  41. /**
  42. * Creates a simple duplex stream.
  43. *
  44. * @param {Function} [callback] A callback which will be executed with each chunk.
  45. * @returns {Stream}
  46. */
  47. noop( callback ) {
  48. if ( !callback ) {
  49. return new PassThrough( { objectMode: true } );
  50. }
  51. return through( { objectMode: true }, ( file, encoding, throughCallback ) => {
  52. callback( file );
  53. throughCallback( null, file );
  54. } );
  55. },
  56. /**
  57. * Saves the files piped into this stream to the `build/` directory.
  58. *
  59. * @param {String} buildDir The `build/` directory path.
  60. * @param {String} format The format of the buildribution (`esnext`, `amd`, or `cjs`).
  61. * @returns {Stream}
  62. */
  63. destBuild( buildDir, format ) {
  64. const destDir = path.join( buildDir, format );
  65. return gulp.dest( destDir );
  66. },
  67. /**
  68. * Creates a function generating convertion streams.
  69. * Used to generate `formats.reduce()` callback where `formats` is an array of formats that should be generated.
  70. *
  71. * @param {String} buildDir The `build/` directory path.
  72. * @returns {Function}
  73. */
  74. getConversionStreamGenerator( buildDir ) {
  75. return ( pipes, format ) => {
  76. const conversionPipes = [];
  77. conversionPipes.push( utils.pickVersionedFile( format ) );
  78. if ( format != 'esnext' ) {
  79. // Convert src files.
  80. const filterSource = gulpFilter( ( file ) => {
  81. return utils.isSourceFile( file ) && utils.isJSFile( file );
  82. }, { restore: true } );
  83. const transpileSource = utils.transpile( format, utils.getBabelOptionsForSource( format ) );
  84. conversionPipes.push(
  85. filterSource,
  86. transpileSource,
  87. utils.fixCoverage(),
  88. filterSource.restore
  89. );
  90. // Convert test files.
  91. const filterTests = gulpFilter( ( file ) => {
  92. return utils.isTestFile( file ) && utils.isJSFile( file );
  93. }, { restore: true } );
  94. const transpileTests = utils.transpile( format, utils.getBabelOptionsForTests( format ) );
  95. conversionPipes.push(
  96. filterTests,
  97. transpileTests,
  98. format === 'amd' ? utils.appendBenderLauncher() : utils.noop(),
  99. filterTests.restore
  100. );
  101. }
  102. conversionPipes.push(
  103. utils.destBuild( buildDir, format ),
  104. utils.noop( ( file ) => {
  105. gutil.log( `Finished writing '${ gutil.colors.cyan( file.path ) }'` );
  106. } )
  107. );
  108. pipes.push( multipipe.apply( null, conversionPipes ) );
  109. return pipes;
  110. };
  111. },
  112. /**
  113. * Transpiles files piped into this stream to the given format (`amd` or `cjs`).
  114. *
  115. * @param {String} format
  116. * @returns {Stream}
  117. */
  118. transpile( format, options ) {
  119. return gulpBabel( options )
  120. .on( 'error', ( err ) => {
  121. gutil.log( gutil.colors.red( `Error (Babel:${ format })` ) );
  122. gutil.log( gutil.colors.red( err.message ) );
  123. console.log( '\n' + err.codeFrame + '\n' );
  124. } );
  125. },
  126. /**
  127. * Adds istanbul ignore to the code created by babel.
  128. *
  129. * @returns {Stream}
  130. */
  131. fixCoverage() {
  132. return replace(
  133. 'return obj && obj.__esModule ? obj :',
  134. '/* istanbul ignore next */\n\t\t' +
  135. 'return obj && obj.__esModule ? obj :' );
  136. },
  137. /**
  138. * Returns an object with Babel options for the source code.
  139. *
  140. * @param {String} format
  141. * @returns {Object} options
  142. */
  143. getBabelOptionsForSource( format ) {
  144. return {
  145. plugins: utils.getBabelPlugins( format ),
  146. // Ensure that all paths ends with '.js' because Require.JS (unlike Common.JS/System.JS)
  147. // will not add it to module names which look like paths.
  148. resolveModuleSource: format == 'cjs' ? utils.resolveModuleSource : utils.appendModuleExtension
  149. };
  150. },
  151. /**
  152. * Returns an object with Babel options for the test code.
  153. *
  154. * @param {String} format
  155. * @returns {Object} options
  156. */
  157. getBabelOptionsForTests( format ) {
  158. return {
  159. plugins: utils.getBabelPlugins( format ),
  160. resolveModuleSource: format == 'cjs' ? utils.resolveModuleSource : utils.appendModuleExtension,
  161. moduleIds: true,
  162. moduleId: 'tests'
  163. };
  164. },
  165. /**
  166. * Returns an array of Babel plugins to use.
  167. *
  168. * @param {String} format
  169. * @returns {Array}
  170. */
  171. getBabelPlugins( format ) {
  172. const babelModuleTranspilers = {
  173. amd: 'amd',
  174. cjs: 'commonjs'
  175. };
  176. const babelModuleTranspiler = babelModuleTranspilers[ format ];
  177. if ( !babelModuleTranspiler ) {
  178. throw new Error( `Incorrect format: ${ format }` );
  179. }
  180. return [
  181. // Note: When plugin is specified by its name, Babel loads it from a context of a
  182. // currently transpiled file (in our case - e.g. from ckeditor5-core/src/foo.js).
  183. // Obviously that fails, since we have all the plugins installed only in ckeditor5/
  184. // and we want to have them only there to avoid installing them dozens of times.
  185. //
  186. // Anyway, I haven't found in the docs that you can also pass a plugin instance here,
  187. // but it works... so let's hope it will.
  188. require( `babel-plugin-transform-es2015-modules-${ babelModuleTranspiler }` )
  189. ];
  190. },
  191. /**
  192. * Appends the {@link #benderLauncherCode} at the end of the file.
  193. *
  194. * @returns {Stream}
  195. */
  196. appendBenderLauncher() {
  197. return through( { objectMode: true }, ( file, encoding, callback ) => {
  198. if ( !file.isNull() ) {
  199. file.contents = new Buffer( file.contents.toString() + utils.benderLauncherCode );
  200. }
  201. callback( null, file );
  202. } );
  203. },
  204. /**
  205. * Allows us to pick one of files suffixed with the format (`__esnext`, `__amd`, or `__cjs`) and removes
  206. * files with other suffixes from the stream.
  207. *
  208. * For example: we have `load__esnext.js`, `load__amd.js` and `load__cjs.js`. After applying this
  209. * transformation when compiling code for a specific format the proper file will be renamed to `load.js`.
  210. * Files not matching a specified format will be removed.
  211. *
  212. * @param {String} format
  213. * @returns {Stream}
  214. */
  215. pickVersionedFile( format ) {
  216. const rejectedFormats = utils.SUPPORTED_FORMATS
  217. .filter( ( item ) => item !== format );
  218. const pickRegexp = new RegExp( `__${ format }$` );
  219. const rejectRegexp = new RegExp( `__(${ rejectedFormats.join( '|' ) }).js$` );
  220. const pick = rename( ( path ) => {
  221. path.basename = path.basename.replace( pickRegexp, '' );
  222. } );
  223. const remove = gulpFilter( ( file ) => !rejectRegexp.test( file.path ) );
  224. return multipipe( pick, remove );
  225. },
  226. /**
  227. * Processes paths of files inside CKEditor5 packages.
  228. *
  229. * * `ckeditor5-xxx/src/foo/bar.js` -> `ckeditor5/xxx/foo/bar.js`
  230. * * `ckeditor5-xxx/tests/foo/bar.js` -> `tests/xxx/foo/bar.js`
  231. *
  232. * @returns {Stream}
  233. */
  234. renamePackageFiles() {
  235. return rename( ( file ) => {
  236. const dirFrags = file.dirname.split( path.sep );
  237. // Validate the input for the clear conscious.
  238. if ( dirFrags[ 0 ].indexOf( 'ckeditor5-' ) !== 0 ) {
  239. throw new Error( 'Path should start with "ckeditor5-".' );
  240. }
  241. dirFrags[ 0 ] = dirFrags[ 0 ].replace( /^ckeditor5-/, '' );
  242. const firstFrag = dirFrags[ 1 ];
  243. if ( firstFrag == 'src' ) {
  244. // Remove 'src/'.
  245. dirFrags.splice( 1, 1 );
  246. // Temporary implementation of the UI lib option. See #88.
  247. if ( dirFrags[ 0 ] == 'ui-default' ) {
  248. dirFrags[ 0 ] = 'ui';
  249. }
  250. // And prepend 'ckeditor5/'.
  251. dirFrags.unshift( 'ckeditor5' );
  252. } else if ( firstFrag == 'tests' ) {
  253. // Remove 'tests/' from the package dir.
  254. dirFrags.splice( 1, 1 );
  255. // And prepend 'tests/'.
  256. dirFrags.unshift( 'tests' );
  257. } else {
  258. throw new Error( 'Path should start with "ckeditor5-*/(src|tests|theme)".' );
  259. }
  260. file.dirname = path.join.apply( null, dirFrags );
  261. } );
  262. },
  263. /**
  264. * Processes paths of files inside the main CKEditor5 package.
  265. *
  266. * * `src/foo/bar.js` -> `ckeditor5/foo/bar.js`
  267. * * `tests/foo/bar.js` -> `tests/ckeditor5/foo/bar.js`
  268. *
  269. * @returns {Stream}
  270. */
  271. renameCKEditor5Files() {
  272. return rename( ( file ) => {
  273. const dirFrags = file.dirname.split( path.sep );
  274. const firstFrag = dirFrags[ 0 ];
  275. if ( firstFrag == 'src' ) {
  276. // Replace 'src/' with 'ckeditor5/'.
  277. // src/path.js -> ckeditor5/path.js
  278. dirFrags.splice( 0, 1, 'ckeditor5' );
  279. } else if ( firstFrag == 'tests' ) {
  280. // Insert 'ckeditor5/' after 'tests/'.
  281. // tests/foo.js -> tests/ckeditor5/foo.js
  282. dirFrags.splice( 1, 0, 'ckeditor5' );
  283. } else {
  284. throw new Error( 'Path should start with "src" or "tests".' );
  285. }
  286. file.dirname = path.join.apply( null, dirFrags );
  287. } );
  288. },
  289. /**
  290. * Appends file extension to file URLs. Tries to not touch named modules.
  291. *
  292. * @param {String} source
  293. * @returns {String}
  294. */
  295. appendModuleExtension( source ) {
  296. if ( /^https?:|\.[\/\\]/.test( source ) && !/\.js$/.test( source ) ) {
  297. return source + '.js';
  298. }
  299. return source;
  300. },
  301. /**
  302. * Checks whether a file is a test file.
  303. *
  304. * @param {Vinyl} file
  305. * @returns {Boolean}
  306. */
  307. isTestFile( file ) {
  308. // TODO this should be based on bender configuration (config.tests.*.paths).
  309. if ( !file.relative.startsWith( 'tests' + path.sep ) ) {
  310. return false;
  311. }
  312. const dirFrags = file.relative.split( path.sep );
  313. return !dirFrags.some( dirFrag => {
  314. return dirFrag.startsWith( '_' ) && dirFrag != '_utils-tests';
  315. } );
  316. },
  317. /**
  318. * Checks whether a file is a source file.
  319. *
  320. * @param {Vinyl} file
  321. * @returns {Boolean}
  322. */
  323. isSourceFile( file ) {
  324. return !utils.isTestFile( file );
  325. },
  326. /**
  327. * Checks whether a file is a JS file.
  328. *
  329. * @param {Vinyl} file
  330. * @returns {Boolean}
  331. */
  332. isJSFile( file ) {
  333. return file.path.endsWith( '.js' );
  334. },
  335. /**
  336. * Finds all CKEditor5 package directories in "node_modules" folder.
  337. *
  338. * @param {String} rootDir A root directory containing "node_modules" folder.
  339. * @returns {Array} Array of ckeditor5-* package directory paths.
  340. */
  341. getPackages( rootDir ) {
  342. // Find all CKEditor5 package directories. Resolve symlinks so we watch real directories
  343. // in order to workaround https://github.com/paulmillr/chokidar/issues/419.
  344. return fs.readdirSync( path.join( rootDir, 'node_modules' ) )
  345. // Look for ckeditor5-* directories.
  346. .filter( fileName => {
  347. return fileName.indexOf( 'ckeditor5-' ) === 0;
  348. } )
  349. // Resolve symlinks and keep only directories.
  350. .map( fileName => {
  351. let filePath = path.join( rootDir, 'node_modules', fileName );
  352. let stat = fs.lstatSync( filePath );
  353. if ( stat.isSymbolicLink() ) {
  354. filePath = fs.realpathSync( filePath );
  355. stat = fs.lstatSync( filePath );
  356. }
  357. if ( stat.isDirectory() ) {
  358. return filePath;
  359. }
  360. // Filter...
  361. return false;
  362. } )
  363. // ...those out.
  364. .filter( filePath => filePath );
  365. },
  366. /**
  367. * Filters theme entry points only from a stream of SCSS files.
  368. *
  369. * @returns {Stream}
  370. */
  371. filterThemeEntryPoints() {
  372. return filter( '**/theme.scss' );
  373. },
  374. /**
  375. * Given the input stream of theme entry-point files (theme.scss), this method:
  376. * 1. Collects paths to entry-point.
  377. * 2. Builds the output CSS theme file using aggregated entry-points.
  378. * 3. Returns a stream containing built CSS theme file.
  379. *
  380. * @param {String} fileName The name of the output CSS theme file.
  381. * @returns {Stream}
  382. */
  383. compileThemes( fileName ) {
  384. const paths = [];
  385. const stream = through.obj( collectThemeEntryPoint, renderThemeFromEntryPoints );
  386. function collectThemeEntryPoint( file, enc, callback ) {
  387. paths.push( file.path );
  388. callback();
  389. }
  390. function renderThemeFromEntryPoints( callback ) {
  391. gutil.log( `Compiling '${ gutil.colors.cyan( fileName ) }' from ${ gutil.colors.cyan( paths.length ) } entry points...` );
  392. // Sort to make sure theme is the very first SASS to build. Otherwise,
  393. // packages using mixins and variables from that theme will throw errors
  394. // because such are not available at this stage of compilation.
  395. const dataToRender = paths.sort( a => -a.indexOf( 'ckeditor5-theme' ) )
  396. // Make sure windows\\style\\paths are preserved.
  397. .map( p => `@import "${ p.replace( /\\/g, '\\\\' ) }";` )
  398. .join( '\n' );
  399. try {
  400. const rendered = sass.renderSync( utils.getSassOptions( dataToRender ) );
  401. stream.push( new gutil.File( {
  402. path: fileName,
  403. contents: new Buffer( rendered.css )
  404. } ) );
  405. callback();
  406. } catch ( err ) {
  407. callback( err );
  408. }
  409. }
  410. return stream;
  411. },
  412. /**
  413. * Parses command line arguments and returns them as a user-friendly hash.
  414. *
  415. * @param {String} dataToRender
  416. * @returns {Object}
  417. */
  418. getSassOptions( dataToRender ) {
  419. return {
  420. data: dataToRender,
  421. sourceMap: true,
  422. sourceMapEmbed: true,
  423. outputStyle: 'expanded',
  424. sourceComments: true
  425. };
  426. },
  427. /**
  428. * Parses command line arguments and returns them as a user-friendly hash.
  429. *
  430. * @returns {Object} options
  431. * @returns {Array} [options.formats] Array of specified output formats ("esnext" or "amd").
  432. * @returns {Boolean} [options.watch] A flag which enables watch mode.
  433. */
  434. parseArguments() {
  435. const options = minimist( process.argv.slice( 2 ), {
  436. string: [
  437. 'formats'
  438. ],
  439. boolean: [
  440. 'watch'
  441. ],
  442. default: {
  443. formats: 'amd',
  444. watch: false
  445. }
  446. } );
  447. options.formats = options.formats.split( ',' );
  448. return options;
  449. },
  450. /**
  451. * Given a stream of `.svg` files it returns a stream containing JavaScript
  452. * icon sprite file.
  453. *
  454. * @returns {Stream}
  455. */
  456. compileIconSprite() {
  457. return sprite( utils.getIconSpriteOptions() );
  458. },
  459. /**
  460. * Returns svg-sprite util options to generate <symbol>-based, JavaScript
  461. * sprite file.
  462. *
  463. * @returns {Object}
  464. */
  465. getIconSpriteOptions() {
  466. return {
  467. shape: {
  468. id: {
  469. // Note: Consider unix/style/paths and windows\\style\\paths.
  470. generator: name => `ck-icon-${ name.match( /([^\/\\]*)\.svg$/ )[ 1 ] }`
  471. },
  472. },
  473. svg: {
  474. xmlDeclaration: false,
  475. doctypeDeclaration: false,
  476. },
  477. mode: {
  478. symbol: {
  479. dest: './', // Flatten symbol/ folder.
  480. inline: true,
  481. render: {
  482. js: {
  483. template: path.join( __dirname, 'iconmanagermodel.tpl' ),
  484. dest: 'iconmanagermodel.js',
  485. }
  486. }
  487. }
  488. }
  489. };
  490. },
  491. /**
  492. * Given a stream of files it returns an array of gulp-mirror streams outputting
  493. * files to `build/[formats]/theme/` directories for each of desired output formats (cjs, amd, etc.).
  494. *
  495. * @param {String} buildDir A path to /build directory.
  496. * @param {Array} formats An array of desired output formats.
  497. * @param {Function} [transformationStream] A stream used to transform files before they're saved to
  498. * desired `build/[formats]/theme` directories. Useful for transpilation.
  499. * @returns {Stream[]} An array of streams.
  500. */
  501. getThemeFormatDestStreams( buildDir, formats, transformationStream ) {
  502. return formats.map( f => {
  503. return pipe(
  504. transformationStream ? transformationStream( f ) : utils.noop(),
  505. gulp.dest( path.join( buildDir, f, 'theme' ) ),
  506. utils.noop( file => {
  507. gutil.log( `Output for ${ gutil.colors.cyan( f ) } is '${ gutil.colors.cyan( file.path ) }'.` );
  508. } )
  509. );
  510. } );
  511. },
  512. /**
  513. * Resolves CommonJS module source path.
  514. *
  515. * @param {String} source Module path passed to require() method.
  516. * @param {String} file Path to a file where require() method is called.
  517. * @returns {String} Fixed module path.
  518. */
  519. resolveModuleSource( source, file ) {
  520. // If path is relative - leave it as is.
  521. if ( !path.isAbsolute( source ) ) {
  522. return source;
  523. }
  524. // Find relative path of test file from cwd directory.
  525. let testFile = path.relative( process.cwd(), file );
  526. // Append `/` as all files uses it as root inside transpiled versions.
  527. testFile = path.join( path.sep, testFile );
  528. // Find relative path from test file to source.
  529. let relativePath = path.relative( path.dirname( testFile ), path.dirname( source ) );
  530. relativePath = path.join( relativePath, path.basename( source ) );
  531. // Convert windows path to posix.
  532. relativePath = relativePath.replace( /\\/g, '/' );
  533. return utils.appendModuleExtension( ( relativePath.startsWith( '../' ) ? '' : './' ) + relativePath );
  534. }
  535. };
  536. module.exports = utils;