8
0

snippetadapter.js 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581
  1. /**
  2. * @license Copyright (c) 2003-2020, CKSource - Frederico Knabben. All rights reserved.
  3. * For licensing, see LICENSE.md or https://ckeditor.com/legal/ckeditor-oss-license
  4. */
  5. /* eslint-env node */
  6. const path = require( 'path' );
  7. const upath = require( 'upath' );
  8. const fs = require( 'fs' );
  9. const minimatch = require( 'minimatch' );
  10. const webpack = require( 'webpack' );
  11. const { bundler, styles } = require( '@ckeditor/ckeditor5-dev-utils' );
  12. const CKEditorWebpackPlugin = require( '@ckeditor/ckeditor5-dev-webpack-plugin' );
  13. const MiniCssExtractPlugin = require( 'mini-css-extract-plugin' );
  14. const TerserPlugin = require( 'terser-webpack-plugin' );
  15. const ProgressBarPlugin = require( 'progress-bar-webpack-plugin' );
  16. const glob = require( 'glob' );
  17. const DEFAULT_LANGUAGE = 'en';
  18. const MULTI_LANGUAGE = 'multi-language';
  19. /**
  20. * @param {Set.<Snippet>} snippets Snippet collection extracted from documentation files.
  21. * @param {Object} options
  22. * @param {Boolean} options.production Whether to build snippets in production mode.
  23. * @param {Array.<String>|undefined} options.allowedSnippets An array that contains glob patterns of snippets that should be built.
  24. * If not specified or if passed the empty array, all snippets will be built.
  25. * @param {Object.<String, Function>} umbertoHelpers
  26. * @returns {Promise}
  27. */
  28. module.exports = function snippetAdapter( snippets, options, umbertoHelpers ) {
  29. const { getSnippetPlaceholder, getSnippetSourcePaths } = umbertoHelpers;
  30. const snippetsDependencies = new Map();
  31. // For each snippet, load its config. If the snippet has defined dependencies, load those as well.
  32. for ( const snippetData of snippets ) {
  33. if ( !snippetData.snippetSources.js ) {
  34. throw new Error( `Missing snippet source for "${ snippetData.snippetName }".` );
  35. }
  36. snippetData.snippetConfig = readSnippetConfig( snippetData.snippetSources.js );
  37. snippetData.snippetConfig.language = snippetData.snippetConfig.language || DEFAULT_LANGUAGE;
  38. // If, in order to work, a snippet requires another snippet to be built, and the other snippet
  39. // isn't included in any guide via `{@snippet ...}`, then that other snippet need to be marked
  40. // as a dependency of the first one. Example – bootstrap UI uses an iframe, and inside that iframe we
  41. // need a JS file. That JS file needs to be built, even though it's not a real snippet (and it's not used
  42. // via {@snippet}).
  43. if ( snippetData.snippetConfig.dependencies ) {
  44. for ( const dependencyName of snippetData.snippetConfig.dependencies ) {
  45. // Do not load the same dependency more than once.
  46. if ( snippetsDependencies.has( dependencyName ) ) {
  47. continue;
  48. }
  49. // Find a root path where to look for the snippet's sources. We just want to pass it through Webpack.
  50. const snippetBasePathRegExp = new RegExp( snippetData.snippetName.replace( /\//g, '\\/' ) + '.*$' );
  51. const snippetBasePath = snippetData.snippetSources.js.replace( snippetBasePathRegExp, '' );
  52. const dependencySnippet = {
  53. snippetSources: getSnippetSourcePaths( snippetBasePath, dependencyName ),
  54. snippetName: dependencyName,
  55. outputPath: snippetData.outputPath,
  56. destinationPath: snippetData.destinationPath,
  57. requiredFor: snippetData
  58. };
  59. if ( !dependencySnippet.snippetSources.js ) {
  60. throw new Error( `Missing snippet source for "${ dependencySnippet.snippetName }".` );
  61. }
  62. dependencySnippet.snippetConfig = readSnippetConfig( dependencySnippet.snippetSources.js );
  63. dependencySnippet.snippetConfig.language = dependencySnippet.snippetConfig.language || DEFAULT_LANGUAGE;
  64. snippetsDependencies.set( dependencyName, dependencySnippet );
  65. }
  66. }
  67. }
  68. // Add all dependencies to the snippet collection.
  69. for ( const snippetData of snippetsDependencies.values() ) {
  70. snippets.add( snippetData );
  71. }
  72. // Remove snippets that do not match to patterns specified in `options.allowedSnippets`.
  73. if ( options.allowedSnippets && options.allowedSnippets.length ) {
  74. filterAllowedSnippets( snippets, options.allowedSnippets );
  75. console.log( `Found ${ snippets.size } matching {@snippet} tags.` );
  76. }
  77. console.log( `Building ${ countUniqueSnippets( snippets ) } snippets...` );
  78. const groupedSnippetsByLanguage = {};
  79. const constantDefinitions = getConstantDefinitions( snippets );
  80. // Group snippets by language. There is no way to build different languages in a single Webpack process.
  81. // Webpack must be called as many times as different languages are being used in snippets.
  82. for ( const snippetData of snippets ) {
  83. // Multi-languages editors must be built separately.
  84. if ( snippetData.snippetConfig.additionalLanguages ) {
  85. snippetData.snippetConfig.additionalLanguages.push( snippetData.snippetConfig.language );
  86. snippetData.snippetConfig.language = MULTI_LANGUAGE;
  87. }
  88. if ( !groupedSnippetsByLanguage[ snippetData.snippetConfig.language ] ) {
  89. groupedSnippetsByLanguage[ snippetData.snippetConfig.language ] = new Set();
  90. }
  91. groupedSnippetsByLanguage[ snippetData.snippetConfig.language ].add( snippetData );
  92. }
  93. // For each language prepare own Webpack configuration.
  94. const webpackConfigs = Object.keys( groupedSnippetsByLanguage )
  95. .map( language => {
  96. return getWebpackConfig( groupedSnippetsByLanguage[ language ], {
  97. language,
  98. production: options.production,
  99. definitions: {
  100. ...( options.definitions || {} ),
  101. ...constantDefinitions
  102. }
  103. } );
  104. } );
  105. let promise = Promise.resolve();
  106. // Nothing to build.
  107. if ( !webpackConfigs.length ) {
  108. return promise;
  109. }
  110. for ( const config of webpackConfigs ) {
  111. promise = promise.then( () => runWebpack( config ) );
  112. }
  113. return promise
  114. .then( () => {
  115. // Group snippets by destination path in order to attach required HTML code and assets (CSS and JS).
  116. const groupedSnippetsByDestinationPath = {};
  117. for ( const snippetData of snippets ) {
  118. if ( !groupedSnippetsByDestinationPath[ snippetData.destinationPath ] ) {
  119. groupedSnippetsByDestinationPath[ snippetData.destinationPath ] = new Set();
  120. }
  121. groupedSnippetsByDestinationPath[ snippetData.destinationPath ].add( snippetData );
  122. }
  123. // For every page that contains at least one snippet, we need to replace Umberto comments with HTML code.
  124. for ( const destinationPath of Object.keys( groupedSnippetsByDestinationPath ) ) {
  125. const snippetsOnPage = groupedSnippetsByDestinationPath[ destinationPath ];
  126. // Assets required for the all snippets.
  127. const cssFiles = [];
  128. const jsFiles = [];
  129. let content = fs.readFileSync( destinationPath ).toString();
  130. for ( const snippetData of snippetsOnPage ) {
  131. // CSS may not be generated by Webpack if a snippet's JS file didn't import any CSS files.
  132. const wasCSSGenerated = fs.existsSync( path.join( snippetData.outputPath, snippetData.snippetName, 'snippet.css' ) );
  133. // If the snippet is a dependency, append JS and CSS to HTML, save to disk and continue.
  134. if ( snippetData.requiredFor ) {
  135. let htmlFile = fs.readFileSync( snippetData.snippetSources.html ).toString();
  136. if ( wasCSSGenerated ) {
  137. htmlFile += '<link rel="stylesheet" href="snippet.css" type="text/css">';
  138. }
  139. htmlFile += '<script src="snippet.js"></script>';
  140. fs.writeFileSync( path.join( snippetData.outputPath, snippetData.snippetName, 'snippet.html' ), htmlFile );
  141. continue;
  142. }
  143. let snippetHTML = fs.readFileSync( snippetData.snippetSources.html ).toString();
  144. if ( snippetHTML.trim() ) {
  145. snippetHTML = snippetHTML.replace( /%BASE_PATH%/g, snippetData.basePath );
  146. snippetHTML = `<div class="live-snippet">${ snippetHTML }</div>`;
  147. }
  148. content = content.replace( getSnippetPlaceholder( snippetData.snippetName ), snippetHTML );
  149. jsFiles.push( path.join( snippetData.basePath, 'assets', 'snippet.js' ) );
  150. jsFiles.push( path.join( snippetData.relativeOutputPath, snippetData.snippetName, 'snippet.js' ) );
  151. cssFiles.push( path.join( snippetData.basePath, 'assets', 'snippet-styles.css' ) );
  152. if ( wasCSSGenerated ) {
  153. cssFiles.unshift( path.join( snippetData.relativeOutputPath, snippetData.snippetName, 'snippet.css' ) );
  154. }
  155. // Additional languages must be imported by the HTML code.
  156. if ( snippetData.snippetConfig.additionalLanguages ) {
  157. snippetData.snippetConfig.additionalLanguages.forEach( language => {
  158. jsFiles.push( path.join( snippetData.relativeOutputPath, 'translations', `${ language }.js` ) );
  159. } );
  160. }
  161. }
  162. const cssImportsHTML = getHTMLImports( cssFiles, importPath => {
  163. return ` <link rel="stylesheet" href="${ importPath }" type="text/css" data-cke="true">`;
  164. } );
  165. const jsImportsHTML = getHTMLImports( jsFiles, importPath => {
  166. return ` <script src="${ importPath }"></script>`;
  167. } );
  168. content = content.replace( '<!--UMBERTO: SNIPPET: CSS-->', cssImportsHTML );
  169. content = content.replace( '<!--UMBERTO: SNIPPET: JS-->', jsImportsHTML );
  170. fs.writeFileSync( destinationPath, content );
  171. }
  172. } )
  173. .then( () => {
  174. console.log( 'Finished building snippets.' );
  175. } );
  176. };
  177. /**
  178. * Removes snippets that names do not match to patterns specified in `allowedSnippets` array.
  179. *
  180. * @param {Set.<Snippet>} snippets Snippet collection extracted from documentation files.
  181. * @param {Array.<String>} allowedSnippets Snippet patterns that should be built.
  182. */
  183. function filterAllowedSnippets( snippets, allowedSnippets ) {
  184. const snippetsToBuild = new Set();
  185. // Find all snippets that matched to specified criteria.
  186. for ( const snippetData of snippets ) {
  187. const shouldBeBuilt = allowedSnippets.some( pattern => {
  188. return minimatch( snippetData.snippetName, pattern ) || snippetData.snippetName.includes( pattern );
  189. } );
  190. if ( shouldBeBuilt ) {
  191. snippetsToBuild.add( snippetData );
  192. }
  193. }
  194. // Find all dependencies that are required for whitelisted snippets.
  195. for ( const snippetData of snippets ) {
  196. if ( snippetsToBuild.has( snippetData ) ) {
  197. continue;
  198. }
  199. if ( snippetData.requiredFor && snippetsToBuild.has( snippetData.requiredFor ) ) {
  200. snippetsToBuild.add( snippetData );
  201. }
  202. }
  203. // Remove snippets that won't be built and aren't dependencies of other snippets.
  204. for ( const snippetData of snippets ) {
  205. if ( !snippetsToBuild.has( snippetData ) ) {
  206. snippets.delete( snippetData );
  207. }
  208. }
  209. }
  210. /**
  211. * Adds constants to the webpack process from external repositories containing `docs/constants.js` files.
  212. *
  213. * @param {Array.<Object>} snippets
  214. * @returns {Object}
  215. */
  216. function getConstantDefinitions( snippets ) {
  217. const knownPaths = new Set();
  218. const constantDefinitions = {};
  219. const constantOrigins = new Map();
  220. for ( const snippet of snippets ) {
  221. if ( !snippet.pageSourcePath ) {
  222. continue;
  223. }
  224. let directory = path.dirname( snippet.pageSourcePath );
  225. while ( !knownPaths.has( directory ) ) {
  226. knownPaths.add( directory );
  227. const absolutePathToConstants = path.join( directory, 'docs', 'constants.js' );
  228. const importPathToConstants = path.relative( __dirname, absolutePathToConstants );
  229. if ( fs.existsSync( absolutePathToConstants ) ) {
  230. const packageConstantDefinitions = require( './' + importPathToConstants );
  231. for ( const constantName in packageConstantDefinitions ) {
  232. const constantValue = packageConstantDefinitions[ constantName ];
  233. if ( constantDefinitions[ constantName ] && constantDefinitions[ constantName ] !== constantValue ) {
  234. throw new Error(
  235. `Definition for the '${ constantName }' constant is duplicated` +
  236. ` (${ importPathToConstants }, ${ constantOrigins.get( constantName ) }).`
  237. );
  238. }
  239. constantDefinitions[ constantName ] = constantValue;
  240. constantOrigins.set( constantName, importPathToConstants );
  241. }
  242. Object.assign( constantDefinitions, packageConstantDefinitions );
  243. }
  244. directory = path.dirname( directory );
  245. }
  246. }
  247. return constantDefinitions;
  248. }
  249. /**
  250. * Prepares configuration for Webpack.
  251. *
  252. * @param {Set.<Snippet>} snippets Snippet collection extracted from documentation files.
  253. * @param {Object} config
  254. * @param {String} config.language Language for the build.
  255. * @param {Boolean} config.production Whether to build for production.
  256. * @param {Object} config.definitions
  257. * @returns {Object}
  258. */
  259. function getWebpackConfig( snippets, config ) {
  260. // Stringify all definitions values. The `DefinePlugin` injects definition values as they are so we need to stringify them,
  261. // so they will become real strings in the generated code. See https://webpack.js.org/plugins/define-plugin/ for more information.
  262. const definitions = {};
  263. for ( const definitionKey in config.definitions ) {
  264. definitions[ definitionKey ] = JSON.stringify( config.definitions[ definitionKey ] );
  265. }
  266. const ckeditorWebpackPluginOptions = {
  267. // All translation files are added to HTML files directly later.
  268. buildAllTranslationsToSeparateFiles: true
  269. };
  270. if ( config.language === MULTI_LANGUAGE ) {
  271. const additionalLanguages = new Set();
  272. // Find all additional languages that must be built.
  273. for ( const snippetData of snippets ) {
  274. for ( const language of snippetData.snippetConfig.additionalLanguages ) {
  275. additionalLanguages.add( language );
  276. }
  277. }
  278. // Pass unique values of `additionalLanguages` to `CKEditorWebpackPlugin`.
  279. ckeditorWebpackPluginOptions.additionalLanguages = [ ...additionalLanguages ];
  280. // Also, set the default language because of the warning that comes from the plugin.
  281. ckeditorWebpackPluginOptions.language = DEFAULT_LANGUAGE;
  282. } else {
  283. ckeditorWebpackPluginOptions.language = config.language;
  284. }
  285. const webpackConfig = {
  286. mode: config.production ? 'production' : 'development',
  287. entry: {},
  288. output: {
  289. filename: '[name]/snippet.js'
  290. },
  291. optimization: {
  292. minimizer: [
  293. new TerserPlugin( {
  294. sourceMap: true,
  295. terserOptions: {
  296. output: {
  297. // Preserve CKEditor 5 license comments.
  298. comments: /^!/
  299. }
  300. },
  301. extractComments: false
  302. } )
  303. ]
  304. },
  305. plugins: [
  306. new MiniCssExtractPlugin( { filename: '[name]/snippet.css' } ),
  307. new CKEditorWebpackPlugin( ckeditorWebpackPluginOptions ),
  308. new webpack.BannerPlugin( {
  309. banner: bundler.getLicenseBanner(),
  310. raw: true
  311. } ),
  312. new webpack.DefinePlugin( definitions ),
  313. new ProgressBarPlugin( {
  314. format: `Building snippets for language "${ config.language }": :percent (:msg)`
  315. } )
  316. ],
  317. // Configure the paths so building CKEditor 5 snippets work even if the script
  318. // is triggered from a directory outside ckeditor5 (e.g. multi-project case).
  319. resolve: {
  320. modules: [
  321. ...getPackageDependenciesPaths(),
  322. ...getModuleResolvePaths()
  323. ]
  324. },
  325. resolveLoader: {
  326. modules: getModuleResolvePaths()
  327. },
  328. module: {
  329. rules: [
  330. {
  331. test: /\.svg$/,
  332. use: [ 'raw-loader' ]
  333. },
  334. {
  335. test: /\.css$/,
  336. use: [
  337. MiniCssExtractPlugin.loader,
  338. 'css-loader',
  339. {
  340. loader: 'postcss-loader',
  341. options: styles.getPostCssConfig( {
  342. themeImporter: {
  343. themePath: require.resolve( '@ckeditor/ckeditor5-theme-lark' )
  344. },
  345. minify: config.production
  346. } )
  347. }
  348. ]
  349. }
  350. ]
  351. }
  352. };
  353. for ( const snippetData of snippets ) {
  354. if ( !webpackConfig.output.path ) {
  355. webpackConfig.output.path = path.normalize( snippetData.outputPath );
  356. }
  357. if ( webpackConfig.entry[ snippetData.snippetName ] ) {
  358. continue;
  359. }
  360. webpackConfig.entry[ snippetData.snippetName ] = snippetData.snippetSources.js;
  361. }
  362. return webpackConfig;
  363. }
  364. /**
  365. * Builds snippets.
  366. *
  367. * @param {Object} webpackConfig
  368. * @returns {Promise}
  369. */
  370. function runWebpack( webpackConfig ) {
  371. return new Promise( ( resolve, reject ) => {
  372. webpack( webpackConfig, ( err, stats ) => {
  373. if ( err ) {
  374. reject( err );
  375. } else if ( stats.hasErrors() ) {
  376. reject( new Error( stats.toString() ) );
  377. } else {
  378. resolve();
  379. }
  380. } );
  381. } );
  382. }
  383. /**
  384. * @returns {Array.<String>}
  385. */
  386. function getModuleResolvePaths() {
  387. return [
  388. path.resolve( __dirname, '..', '..', 'node_modules' ),
  389. 'node_modules'
  390. ];
  391. }
  392. /**
  393. * Returns an array that contains paths to packages' dependencies.
  394. * The snippet adapter should use packages' dependencies instead of the documentation builder dependencies.
  395. *
  396. * See #7916.
  397. *
  398. * @returns {Array.<String>}
  399. */
  400. function getPackageDependenciesPaths() {
  401. const globOptions = {
  402. cwd: path.resolve( __dirname, '..', '..' ),
  403. absolute: true
  404. };
  405. return glob.sync( 'packages/*/node_modules', globOptions )
  406. .concat( glob.sync( 'external/*/packages/*/node_modules', globOptions ) )
  407. .map( p => path.normalize( p ) );
  408. }
  409. /**
  410. * Reads the snippet's configuration.
  411. *
  412. * @param {String} snippetSourcePath An absolute path to the file.
  413. * @returns {Object}
  414. */
  415. function readSnippetConfig( snippetSourcePath ) {
  416. const snippetSource = fs.readFileSync( snippetSourcePath ).toString();
  417. const configSourceMatch = snippetSource.match( /\n\/\* config ([\s\S]+?)\*\// );
  418. if ( !configSourceMatch ) {
  419. return {};
  420. }
  421. return JSON.parse( configSourceMatch[ 1 ] );
  422. }
  423. /**
  424. * Removes duplicated entries specified in `files` array, unifies path separators to always be `/`
  425. * and then maps those entries using `mapFunction`.
  426. *
  427. * @param {Array.<String>} files Paths collection.
  428. * @param {Function} mapFunction Function that should return a string.
  429. * @returns {String}
  430. */
  431. function getHTMLImports( files, mapFunction ) {
  432. return [ ...new Set( files ) ]
  433. .map( path => upath.normalize( path ) )
  434. .map( mapFunction )
  435. .join( '\n' )
  436. .replace( /^\s+/, '' );
  437. }
  438. /**
  439. * Returns a number of unique snippet names that will be built.
  440. *
  441. * @param {Set.<Snippet>} snippets Snippet collection extracted from documentation files.
  442. * @returns {Number}
  443. */
  444. function countUniqueSnippets( snippets ) {
  445. return new Set( Array.from( snippets, snippet => snippet.snippetName ) ).size;
  446. }
  447. /**
  448. * @typedef {Object} Snippet
  449. *
  450. * @property {SnippetSource} snippetSources Sources of the snippet.
  451. *
  452. * @property {String} snippetName Name of the snippet. Defined directly after `@snippet` tag.
  453. *
  454. * @property {String} outputPath An absolute path where to write file produced by the `snippetAdapter`.
  455. *
  456. * @property {String} destinationPath An absolute path to the file where the snippet is being used.
  457. *
  458. * @property {SnippetConfiguration} snippetConfig={} Additional configuration of the snippet. It's being read from the snippet's source.
  459. *
  460. * @property {String} [basePath] Relative path from the processed file to the root of the documentation.
  461. *
  462. * @property {String} [relativeOutputPath] The same like `basePath` but for the output path (where processed file will be saved).
  463. *
  464. * @property {Snippet|undefined} [requiredFor] If the value is instance of `Snippet`, current snippet requires
  465. * the snippet defined as `requiredFor` to work.
  466. */
  467. /**
  468. * @typedef {Object} SnippetSource
  469. *
  470. * @property {String} html An absolute path to the HTML sample.
  471. *
  472. * @property {String} css An absolute path to the CSS sample.
  473. *
  474. * @property {String} js An absolute path to the JS sample.
  475. */
  476. /**
  477. * @typedef {Object} SnippetConfiguration
  478. *
  479. * @property {String} [language] A language that will be used for building the editor.
  480. *
  481. * @property {Array.<String>} [dependencies] Names of samples that are required to working.
  482. *
  483. * @property {Array.<String>} [additionalLanguages] Additional languages that are required by the snippet.
  484. */