9560614935

Committed 18 Jun 2024 07:12AM UTC coverage: 90.902% (-1.7%) from 92.59%

Build # 9560614935

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #1803 from kennyhyun/fix/utils-error-logging

Fix/utils error logging

Run Details

941 of 1083 branches covered (86.89%)

Branch coverage included in aggregate %.

6 of 6 new or added lines in 2 files covered. (100.0%)

37 existing lines in 3 files now uncovered.

2536 of 2742 relevant lines covered (92.49%)

99.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.78

/lib/packExternalModules.js

'use strict';

const BbPromise = require('bluebird');
const _ = require('lodash');
const path = require('path');
const fse = require('fs-extra');
const findWorkspaceRoot = require('find-yarn-workspace-root');

const Packagers = require('./packagers');
const { isProviderGoogle } = require('./utils');

function rebaseFileReferences(pathToPackageRoot, moduleVersion) {
  if (/^(?:file:[^/]{2}|\.\/|\.\.\/)/.test(moduleVersion)) {
    const filePath = _.replace(moduleVersion, /^file:/, '');
    return _.replace(
      `${_.startsWith(moduleVersion, 'file:') ? 'file:' : ''}${pathToPackageRoot}/${filePath}`,
      /\\/g,
      '/'
    );
  }

  return moduleVersion;
}

/**
 * Add the given modules to a package json's dependencies.
 */
function addModulesToPackageJson(externalModules, packageJson, pathToPackageRoot) {
  _.forEach(externalModules.sort(), externalModule => {
    const splitModule = _.split(externalModule, '@');
    // If we have a scoped module we have to re-add the @
    if (_.startsWith(externalModule, '@')) {
      splitModule.splice(0, 1);
      splitModule[0] = '@' + splitModule[0];
    }
    let moduleVersion = _.join(_.tail(splitModule), '@');
    // We have to rebase file references to the target package.json
    moduleVersion = rebaseFileReferences(pathToPackageRoot, moduleVersion);
    packageJson.dependencies = packageJson.dependencies || {};
    packageJson.dependencies[_.first(splitModule)] = moduleVersion;
  });
}

/**
 * Remove a given list of excluded modules from a module list
 * @this - The active plugin instance
 */
function removeExcludedModules(modules, packageForceExcludes, log) {
  // eslint-disable-next-line lodash/prefer-immutable-method
  const excludedModules = _.remove(modules, externalModule => {
    const splitModule = _.split(externalModule, '@');
    // If we have a scoped module we have to re-add the @
    if (_.startsWith(externalModule, '@')) {
      splitModule.splice(0, 1);
      splitModule[0] = '@' + splitModule[0];
    }
    const moduleName = _.first(splitModule);
    return _.includes(packageForceExcludes, moduleName);
  });

  if (log && !_.isEmpty(excludedModules)) {
    if (this.log) {
      this.log(`Excluding external modules: ${_.join(excludedModules, ', ')}`);
    } else {
      this.serverless.cli.log(`Excluding external modules: ${_.join(excludedModules, ', ')}`);
    }
  }
}

/**
 * Resolve the needed versions of production dependencies for external modules.
 * @this - The active plugin instance
 */
function getProdModules(externalModules, packagePath, nodeModulesRelativeDir, dependencyGraph, forceExcludes) {
  const packageJsonPath = path.join(process.cwd(), packagePath);
  const packageJson = require(packageJsonPath);
  const prodModules = [];

  // only process the module stated in dependencies section
  if (!packageJson.dependencies) {
    return [];
  }

  // Get versions of all transient modules
  _.forEach(externalModules, module => {
    let moduleVersion = packageJson.dependencies[module.external];

    if (moduleVersion) {
      prodModules.push(`${module.external}@${moduleVersion}`);

      let nodeModulesBase = path.join(path.dirname(path.join(process.cwd(), packagePath)), 'node_modules');

      if (nodeModulesRelativeDir) {
        const customNodeModulesDir = path.join(process.cwd(), nodeModulesRelativeDir, 'node_modules');

        if (fse.pathExistsSync(customNodeModulesDir)) {
          nodeModulesBase = customNodeModulesDir;
        } else {
          if (this.log) {
            this.log.warning(`${customNodeModulesDir} dose not exist. Please check nodeModulesRelativeDir setting`);
          } else {
            this.serverless.cli.log(
              `WARNING: ${customNodeModulesDir} dose not exist. Please check nodeModulesRelativeDir setting`
            );
          }
        }
      }

      // Check if the module has any peer dependencies and include them too
      try {
        const modulePackagePath = path.join(nodeModulesBase, module.external, 'package.json');

        const peerDependencies = require(modulePackagePath).peerDependencies;
        if (!_.isEmpty(peerDependencies)) {
          if (this.log) {
            this.log.verbose(`Adding explicit peers for dependency ${module.external}`);
          } else {
            this.options.verbose && this.serverless.cli.log(`Adding explicit peers for dependency ${module.external}`);
          }

          const peerDependenciesMeta = require(modulePackagePath).peerDependenciesMeta;

          if (!_.isEmpty(peerDependenciesMeta)) {
            _.forEach(peerDependencies, (value, key) => {
              if (peerDependenciesMeta[key] && peerDependenciesMeta[key].optional === true) {
                if (this.log) {
                  this.log.verbose(
                    `Skipping peers dependency ${key} for dependency ${module.external} because it's optional`
                  );
                } else {
                  this.options.verbose &&
                    this.serverless.cli.log(
                      `Skipping peers dependency ${key} for dependency ${module.external} because it's optional`
                    );
                }
                _.unset(peerDependencies, key);
              }
            });
          }

          if (!_.isEmpty(peerDependencies)) {
            const peerModules = getProdModules.call(
              this,
              _.map(peerDependencies, (value, key) => ({ external: key })),
              packagePath,
              nodeModulesRelativeDir,
              dependencyGraph,
              forceExcludes
            );
            Array.prototype.push.apply(prodModules, peerModules);
          }
        }
      } catch (e) {
        if (this.log) {
          this.log.warning(
            `Could not check for peer dependencies of ${module.external}. Set nodeModulesRelativeDir if node_modules is in different directory.`
          );
        } else {
          this.serverless.cli.log(
            `WARNING: Could not check for peer dependencies of ${module.external}. Set nodeModulesRelativeDir if node_modules is in different directory.`
          );
        }
      }
    } else {
      if (!packageJson.devDependencies || !packageJson.devDependencies[module.external]) {
        // Add transient dependencies if they appear not in the service's dev dependencies
        const originInfo = _.get(dependencyGraph, 'dependencies', {})[module.origin] || {};
        moduleVersion = _.get(_.get(originInfo, 'dependencies', {})[module.external], 'version');
        if (!moduleVersion) {
          // eslint-disable-next-line lodash/path-style
          moduleVersion = _.get(dependencyGraph, ['dependencies', module.external, 'version']);
        }
        if (!moduleVersion) {
          if (this.log) {
            this.log.warning(`Could not determine version of module ${module.external}`);
          } else {
            this.serverless.cli.log(`WARNING: Could not determine version of module ${module.external}`);
          }
        }
        prodModules.push(moduleVersion ? `${module.external}@${moduleVersion}` : module.external);
      } else if (
        packageJson.devDependencies &&
        packageJson.devDependencies[module.external] &&
        !_.includes(forceExcludes, module.external)
      ) {
        // To minimize the chance of breaking setups we whitelist packages available on AWS here. These are due to the previously missing check
        // most likely set in devDependencies and should not lead to an error now.
        const ignoredDevDependencies = ['aws-sdk'];

        if (!_.includes(ignoredDevDependencies, module.external)) {
          // Runtime dependency found in devDependencies but not forcefully excluded
          if (this.log) {
            this.log.error(
              `Runtime dependency '${module.external}' found in devDependencies. Move it to dependencies or use forceExclude to explicitly exclude it.`
            );
          } else {
            this.serverless.cli.log(
              `ERROR: Runtime dependency '${module.external}' found in devDependencies. Move it to dependencies or use forceExclude to explicitly exclude it.`
            );
          }
          throw new this.serverless.classes.Error(`Serverless-webpack dependency error: ${module.external}.`);
        }
        if (this.log) {
          this.log.verbose(
            `Runtime dependency '${module.external}' found in devDependencies. It has been excluded automatically.`
          );
        } else {
          this.options.verbose &&
            this.serverless.cli.log(
              `INFO: Runtime dependency '${module.external}' found in devDependencies. It has been excluded automatically.`
            );
        }
      }
    }
  });

  return prodModules;
}

module.exports = {
  /**
   * We need a performant algorithm to install the packages for each single
   * function (in case we package individually).
   * (1) We fetch ALL packages needed by ALL functions in a first step
   * and use this as a base npm checkout. The checkout will be done to a
   * separate temporary directory with a package.json that contains everything.
   * (2) For each single compile we copy the whole node_modules to the compile
   * directory and create a (function) compile specific package.json and store
   * it in the compile directory. Now we start npm again there, and npm will just
   * remove the superfluous packages and optimize the remaining dependencies.
   * This will utilize the npm cache at its best and give us the needed results
   * and performance.
   */
  packExternalModules() {
    if (this.skipCompile) {
      return BbPromise.resolve();
    }

    const stats = this.compileStats;

    const includes = this.configuration.includeModules;

    if (!includes) {
      return BbPromise.resolve();
    }
    if (this.log) {
      this.log.verbose('Packing external modules');
      this.progress.get('webpack').notice('[Webpack] Packing external modules');
    }

    // Read plugin configuration
    const packageForceIncludes = _.get(includes, 'forceInclude', []);
    const packageForceExcludes = _.get(includes, 'forceExclude', []);
    const packagePath = includes.packagePath || './package.json';
    const nodeModulesRelativeDir = includes.nodeModulesRelativeDir;
    const packageJsonPath = path.join(process.cwd(), packagePath);
    const packageScripts = _.reduce(
      this.configuration.packagerOptions.scripts || [],
      (__, script, index) => {
        __[`script${index}`] = script;
        return __;
      },
      {}
    );

    // Determine and create packager
    return BbPromise.try(() => Packagers.get.call(this, this.configuration.packager)).then(packager => {
      // Fetch needed original package.json sections
      const sectionNames = packager.copyPackageSectionNames(this.configuration.packagerOptions);
      const packageJson = this.serverless.utils.readFileSync(packageJsonPath);
      const packageSections = _.pick(packageJson, sectionNames);
      if (!_.isEmpty(packageSections)) {
        if (this.log) {
          this.log.verbose(`Using package.json sections ${_.join(_.keys(packageSections), ', ')}`);
        } else {
          this.options.verbose &&
            this.serverless.cli.log(`Using package.json sections ${_.join(_.keys(packageSections), ', ')}`);
        }
      }

      // Get first level dependency graph
      if (this.log) {
        this.log.verbose(`Fetch dependency graph from ${packageJsonPath}`);
      } else {
        this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
      }

      return packager
        .getProdDependencies(path.dirname(packageJsonPath), 1, this.configuration.packagerOptions)
        .then(dependencyGraph => {
          const problems = _.get(dependencyGraph, 'problems', []);
          if (this.options.verbose && !_.isEmpty(problems)) {
            if (this.log) {
              this.log.verbose(`Ignoring ${_.size(problems)} NPM errors:`);
            } else {
              this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
            }
            _.forEach(problems, problem => {
              if (this.log) {
                this.log.verbose(`=> ${problem}`);
              } else {
                this.serverless.cli.log(`=> ${problem}`);
              }
            });
          }

          // (1) Generate dependency composition
          const compositeModules = _.uniq(
            _.flatMap(stats.stats, compileStats => {
              const externalModules = _.concat(
                compileStats.externalModules,
                _.map(packageForceIncludes, whitelistedPackage => ({
                  external: whitelistedPackage
                }))
              );
              return getProdModules.call(
                this,
                externalModules,
                packagePath,
                nodeModulesRelativeDir,
                dependencyGraph,
                packageForceExcludes
              );
            })
          );
          removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);

          if (_.isEmpty(compositeModules)) {
            // The compiled code does not reference any external modules at all
            if (this.log) {
              this.log('No external modules needed');
            } else {
              this.serverless.cli.log('No external modules needed');
            }
            return BbPromise.resolve();
          }

          // (1.a) Install all needed modules
          const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
          const compositePackageJson = path.join(compositeModulePath, 'package.json');

          // (1.a.1) Create a package.json
          const compositePackage = _.defaults(
            {
              name: this.serverless.service.service,
              version: '1.0.0',
              description: `Packaged externals for ${this.serverless.service.service}`,
              private: true,
              scripts: packageScripts
            },
            packageSections
          );
          const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));
          addModulesToPackageJson(compositeModules, compositePackage, relPath);
          this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));

          // (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
          const packagerOptions = this.configuration.packagerOptions || {};
          const packageLockPath = path.join(
            findWorkspaceRoot(path.dirname(packageJsonPath)) || path.dirname(packageJsonPath),
            packagerOptions.lockFile || packager.lockfileName
          );
          let hasPackageLock = false;
          return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))
            .then(exists => {
              if (exists) {
                if (this.log) {
                  this.log('Package lock found - Using locked versions');
                } else {
                  this.serverless.cli.log('Package lock found - Using locked versions');
                }
                try {
                  let packageLockFile = this.serverless.utils.readFileSync(packageLockPath);
                  packageLockFile = packager.rebaseLockfile(relPath, packageLockFile);
                  if (_.isObject(packageLockFile)) {
                    packageLockFile = JSON.stringify(packageLockFile, null, 2);
                  }

                  this.serverless.utils.writeFileSync(
                    path.join(compositeModulePath, packager.lockfileName),
                    packageLockFile
                  );
                  hasPackageLock = true;
                } catch (err) {
                  if (this.log) {
                    this.log.warning(`Could not read lock file: ${err.message}`);
                  } else {
                    this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);
                  }
                }
              }
              return BbPromise.resolve();
            })
            .then(() => {
              const start = _.now();
              if (this.log) {
                this.log('Packing external modules: ' + compositeModules.join(', '));
              } else {
                this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
              }

              return packager.getPackagerVersion(compositeModulePath).then(version => {
                return packager
                  .install(compositeModulePath, this.configuration.packagerOptions, version)
                  .then(() => {
                    if (this.log) {
                      this.log.verbose(`Package took [${_.now() - start} ms]`);
                    } else {
                      this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`);
                    }
                    return null;
                  })
                  .return(stats.stats);
              });
            })
            .mapSeries(compileStats => {
              const modulePath = compileStats.outputPath;

              // Create package.json
              const modulePackageJson = path.join(modulePath, 'package.json');
              const modulePackage = _.defaults(
                {
                  name: this.serverless.service.service,
                  version: '1.0.0',
                  description: `Packaged externals for ${this.serverless.service.service}`,
                  private: true,
                  scripts: packageScripts,
                  dependencies: {}
                },
                packageSections
              );
              const prodModules = getProdModules.call(
                this,
                _.concat(
                  compileStats.externalModules,
                  _.map(packageForceIncludes, whitelistedPackage => ({
                    external: whitelistedPackage
                  }))
                ),
                packagePath,
                nodeModulesRelativeDir,
                dependencyGraph,
                packageForceExcludes
              );
              removeExcludedModules.call(this, prodModules, packageForceExcludes);
              const relPath = path.relative(modulePath, path.dirname(packageJsonPath));
              addModulesToPackageJson(prodModules, modulePackage, relPath);
              this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));

              // GOOGLE: Copy modules only if not google-cloud-functions
              //         GCF Auto installs the package json
              if (isProviderGoogle(this.serverless)) {
                return BbPromise.resolve();
              }

              const startCopy = _.now();
              return BbPromise.try(() => {
                // Only copy dependency modules if demanded by packager
                if (packager.mustCopyModules) {
                  return BbPromise.fromCallback(callback =>
                    fse.copy(
                      path.join(compositeModulePath, 'node_modules'),
                      path.join(modulePath, 'node_modules'),
                      callback
                    )
                  );
                }
                return BbPromise.resolve();
              })
                .then(() =>
                  hasPackageLock
                    ? BbPromise.fromCallback(callback =>
                        fse.copy(
                          path.join(compositeModulePath, packager.lockfileName),
                          path.join(modulePath, packager.lockfileName),
                          callback
                        )
                      )
                    : BbPromise.resolve()
                )
                .tap(() => {
                  if (this.log) {
                    this.log.verbose(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`);
                  } else {
                    this.options.verbose &&
                      this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`);
                  }
                })
                .then(() => {
                  // Prune extraneous packages - removes not needed ones
                  const startPrune = _.now();
                  return packager.getPackagerVersion(modulePath).then(version => {
                    return packager.prune(modulePath, this.configuration.packagerOptions, version).tap(() => {
                      if (this.log) {
                        this.log.verbose(`Prune: ${modulePath} [${_.now() - startPrune} ms]`);
                      } else {
                        this.options.verbose &&
                          this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`);
                      }
                    });
                  });
                })
                .then(() => {
                  // Prune extraneous packages - removes not needed ones
                  const startRunScripts = _.now();
                  return packager.runScripts(modulePath, _.keys(packageScripts)).tap(() => {
                    if (this.log) {
                      this.log.verbose(`Run scripts: ${modulePath} [${_.now() - startRunScripts} ms]`);
                    } else {
                      this.options.verbose &&
                        this.serverless.cli.log(`Run scripts: ${modulePath} [${_.now() - startRunScripts} ms]`);
                    }
                  });
                });
            })
            .return();
        });
    });
  }
};

1	'use strict';	2✔
2		2✔
3	const BbPromise = require('bluebird');	2✔
4	const _ = require('lodash');	2✔
5	const path = require('path');	2✔
6	const fse = require('fs-extra');	2✔
7	const findWorkspaceRoot = require('find-yarn-workspace-root');	2✔
8		2✔
9	const Packagers = require('./packagers');	2✔
10	const { isProviderGoogle } = require('./utils');	2✔
11		2✔
12	function rebaseFileReferences(pathToPackageRoot, moduleVersion) {	246✔
13	if (/^(?:file:[^/]{2}\|\.\/\|\.\.\/)/.test(moduleVersion)) {	246✔
14	const filePath = _.replace(moduleVersion, /^file:/, '');	4✔
15	return _.replace(	4✔
16	`${_.startsWith(moduleVersion, 'file:') ? 'file:' : ''}${pathToPackageRoot}/${filePath}`,	4!
17	/\\/g,	4✔
18	'/'	4✔
19	);	4✔
20	}	4✔
21		242✔
22	return moduleVersion;	242✔
23	}	242✔
24		2✔
25	/**	2✔
26	* Add the given modules to a package json's dependencies.	2✔
27	*/	2✔
28	function addModulesToPackageJson(externalModules, packageJson, pathToPackageRoot) {	78✔
29	_.forEach(externalModules.sort(), externalModule => {	78✔
30	const splitModule = _.split(externalModule, '@');	246✔
31	// If we have a scoped module we have to re-add the @	246✔
32	if (_.startsWith(externalModule, '@')) {	246✔
33	splitModule.splice(0, 1);	62✔
34	splitModule[0] = '@' + splitModule[0];	62✔
35	}	62✔
36	let moduleVersion = _.join(_.tail(splitModule), '@');	246✔
37	// We have to rebase file references to the target package.json	246✔
38	moduleVersion = rebaseFileReferences(pathToPackageRoot, moduleVersion);	246✔
39	packageJson.dependencies = packageJson.dependencies \|\| {};	246✔
40	packageJson.dependencies[_.first(splitModule)] = moduleVersion;	246✔
41	});	78✔
42	}	78✔
43		2✔
44	/**	2✔
45	* Remove a given list of excluded modules from a module list	2✔
46	* @this - The active plugin instance	2✔
47	*/	2✔
48	function removeExcludedModules(modules, packageForceExcludes, log) {	80✔
49	// eslint-disable-next-line lodash/prefer-immutable-method	80✔
50	const excludedModules = _.remove(modules, externalModule => {	80✔
51	const splitModule = _.split(externalModule, '@');	250✔
52	// If we have a scoped module we have to re-add the @	250✔
53	if (_.startsWith(externalModule, '@')) {	250✔
54	splitModule.splice(0, 1);	62✔
55	splitModule[0] = '@' + splitModule[0];	62✔
56	}	62✔
57	const moduleName = _.first(splitModule);	250✔
58	return _.includes(packageForceExcludes, moduleName);	250✔
59	});	80✔
60		80✔
61	if (log && !_.isEmpty(excludedModules)) {	80✔
62	if (this.log) {	2!
63	this.log(`Excluding external modules: ${_.join(excludedModules, ', ')}`);	×
64	} else {	2✔
65	this.serverless.cli.log(`Excluding external modules: ${_.join(excludedModules, ', ')}`);	2✔
66	}	2✔
67	}	2✔
68	}	80✔
69		2✔
70	/**	2✔
71	* Resolve the needed versions of production dependencies for external modules.	2✔
72	* @this - The active plugin instance	2✔
73	*/	2✔
74	function getProdModules(externalModules, packagePath, nodeModulesRelativeDir, dependencyGraph, forceExcludes) {	94✔
75	const packageJsonPath = path.join(process.cwd(), packagePath);	94✔
76	const packageJson = require(packageJsonPath);	94✔
77	const prodModules = [];	94✔
78		94✔
79	// only process the module stated in dependencies section	94✔
80	if (!packageJson.dependencies) {	94!
81	return [];	×
82	}	×
83		94✔
84	// Get versions of all transient modules	94✔
85	_.forEach(externalModules, module => {	94✔
86	let moduleVersion = packageJson.dependencies[module.external];	260✔
87		260✔
88	if (moduleVersion) {	260✔
89	prodModules.push(`${module.external}@${moduleVersion}`);	234✔
90		234✔
91	let nodeModulesBase = path.join(path.dirname(path.join(process.cwd(), packagePath)), 'node_modules');	234✔
92		234✔
93	if (nodeModulesRelativeDir) {	234✔
94	const customNodeModulesDir = path.join(process.cwd(), nodeModulesRelativeDir, 'node_modules');	12✔
95		12✔
96	if (fse.pathExistsSync(customNodeModulesDir)) {	12✔
97	nodeModulesBase = customNodeModulesDir;	12✔
98	} else {	12!
99	if (this.log) {	×
100	this.log.warning(`${customNodeModulesDir} dose not exist. Please check nodeModulesRelativeDir setting`);	×
101	} else {	×
102	this.serverless.cli.log(	×
103	`WARNING: ${customNodeModulesDir} dose not exist. Please check nodeModulesRelativeDir setting`	×
104	);	×
105	}	×
106	}	×
107	}	12✔
108		234✔
109	// Check if the module has any peer dependencies and include them too	234✔
110	try {	234✔
111	const modulePackagePath = path.join(nodeModulesBase, module.external, 'package.json');	234✔
112		234✔
113	const peerDependencies = require(modulePackagePath).peerDependencies;	234✔
114	if (!_.isEmpty(peerDependencies)) {	234✔
115	if (this.log) {	12!
116	this.log.verbose(`Adding explicit peers for dependency ${module.external}`);	×
117	} else {	12✔
118	this.options.verbose && this.serverless.cli.log(`Adding explicit peers for dependency ${module.external}`);	12✔
119	}	12✔
120		12✔
121	const peerDependenciesMeta = require(modulePackagePath).peerDependenciesMeta;	12✔
122		12✔
123	if (!_.isEmpty(peerDependenciesMeta)) {	12✔
124	_.forEach(peerDependencies, (value, key) => {	8✔
125	if (peerDependenciesMeta[key] && peerDependenciesMeta[key].optional === true) {	12✔
126	if (this.log) {	4!
127	this.log.verbose(	×
128	`Skipping peers dependency ${key} for dependency ${module.external} because it's optional`	×
129	);	×
130	} else {	4✔
131	this.options.verbose &&	4✔
132	this.serverless.cli.log(	4✔
133	`Skipping peers dependency ${key} for dependency ${module.external} because it's optional`	4✔
134	);	4✔
135	}	4✔
136	_.unset(peerDependencies, key);	4✔
137	}	4✔
138	});	8✔
139	}	8✔
140		12✔
141	if (!_.isEmpty(peerDependencies)) {	12✔
142	const peerModules = getProdModules.call(	12✔
143	this,	12✔
144	_.map(peerDependencies, (value, key) => ({ external: key })),	12✔
145	packagePath,	12✔
146	nodeModulesRelativeDir,	12✔
147	dependencyGraph,	12✔
148	forceExcludes	12✔
149	);	12✔
150	Array.prototype.push.apply(prodModules, peerModules);	12✔
151	}	12✔
152	}	12✔
153	} catch (e) {	234✔
154	if (this.log) {	98!
UNCOV 155	this.log.warning(	×
UNCOV 156	`Could not check for peer dependencies of ${module.external}. Set nodeModulesRelativeDir if node_modules is in different directory.`	×
UNCOV 157	);	×
158	} else {	98✔
159	this.serverless.cli.log(	98✔
160	`WARNING: Could not check for peer dependencies of ${module.external}. Set nodeModulesRelativeDir if node_modules is in different directory.`	98✔
161	);	98✔
162	}	98✔
163	}	98✔
164	} else {	260✔
165	if (!packageJson.devDependencies \|\| !packageJson.devDependencies[module.external]) {	26✔
166	// Add transient dependencies if they appear not in the service's dev dependencies	16✔
167	const originInfo = _.get(dependencyGraph, 'dependencies', {})[module.origin] \|\| {};	16✔
168	moduleVersion = _.get(_.get(originInfo, 'dependencies', {})[module.external], 'version');	16✔
169	if (!moduleVersion) {	16✔
170	// eslint-disable-next-line lodash/path-style	16✔
171	moduleVersion = _.get(dependencyGraph, ['dependencies', module.external, 'version']);	16✔
172	}	16✔
173	if (!moduleVersion) {	16✔
174	if (this.log) {	12!
175	this.log.warning(`Could not determine version of module ${module.external}`);	×
176	} else {	12✔
177	this.serverless.cli.log(`WARNING: Could not determine version of module ${module.external}`);	12✔
178	}	12✔
179	}	12✔
180	prodModules.push(moduleVersion ? `${module.external}@${moduleVersion}` : module.external);	16✔
181	} else if (	26✔
182	packageJson.devDependencies &&	10✔
183	packageJson.devDependencies[module.external] &&	10✔
184	!_.includes(forceExcludes, module.external)	10✔
185	) {	10✔
186	// To minimize the chance of breaking setups we whitelist packages available on AWS here. These are due to the previously missing check	6✔
187	// most likely set in devDependencies and should not lead to an error now.	6✔
188	const ignoredDevDependencies = ['aws-sdk'];	6✔
189		6✔
190	if (!_.includes(ignoredDevDependencies, module.external)) {	6✔
191	// Runtime dependency found in devDependencies but not forcefully excluded	2✔
192	if (this.log) {	2!
193	this.log.error(	×
194	`Runtime dependency '${module.external}' found in devDependencies. Move it to dependencies or use forceExclude to explicitly exclude it.`	×
195	);	×
196	} else {	2✔
197	this.serverless.cli.log(	2✔
198	`ERROR: Runtime dependency '${module.external}' found in devDependencies. Move it to dependencies or use forceExclude to explicitly exclude it.`	2✔
199	);	2✔
200	}	2✔
201	throw new this.serverless.classes.Error(`Serverless-webpack dependency error: ${module.external}.`);	2✔
202	}	2✔
203	if (this.log) {	6!
UNCOV 204	this.log.verbose(	×
UNCOV 205	`Runtime dependency '${module.external}' found in devDependencies. It has been excluded automatically.`	×
UNCOV 206	);	×
207	} else {	6✔
208	this.options.verbose &&	4✔
209	this.serverless.cli.log(	4✔
210	`INFO: Runtime dependency '${module.external}' found in devDependencies. It has been excluded automatically.`	4✔
211	);	4✔
212	}	4✔
213	}	6✔
214	}	26✔
215	});	94✔
216		94✔
217	return prodModules;	94✔
218	}	94✔
219		2✔
220	module.exports = {	2✔
221	/**	2✔
222	* We need a performant algorithm to install the packages for each single	2✔
223	* function (in case we package individually).	2✔
224	* (1) We fetch ALL packages needed by ALL functions in a first step	2✔
225	* and use this as a base npm checkout. The checkout will be done to a	2✔
226	* separate temporary directory with a package.json that contains everything.	2✔
227	* (2) For each single compile we copy the whole node_modules to the compile	2✔
228	* directory and create a (function) compile specific package.json and store	2✔
229	* it in the compile directory. Now we start npm again there, and npm will just	2✔
230	* remove the superfluous packages and optimize the remaining dependencies.	2✔
231	* This will utilize the npm cache at its best and give us the needed results	2✔
232	* and performance.	2✔
233	*/	2✔
234	packExternalModules() {	2✔
235	if (this.skipCompile) {	50✔
236	return BbPromise.resolve();	2✔
237	}	2✔
238		48✔
239	const stats = this.compileStats;	48✔
240		48✔
241	const includes = this.configuration.includeModules;	48✔
242		48✔
243	if (!includes) {	50✔
244	return BbPromise.resolve();	2✔
245	}	2✔
246	if (this.log) {	50!
UNCOV 247	this.log.verbose('Packing external modules');	×
UNCOV 248	this.progress.get('webpack').notice('[Webpack] Packing external modules');	×
UNCOV 249	}	✔
250		46✔
251	// Read plugin configuration	46✔
252	const packageForceIncludes = _.get(includes, 'forceInclude', []);	46✔
253	const packageForceExcludes = _.get(includes, 'forceExclude', []);	46✔
254	const packagePath = includes.packagePath \|\| './package.json';	50✔
255	const nodeModulesRelativeDir = includes.nodeModulesRelativeDir;	50✔
256	const packageJsonPath = path.join(process.cwd(), packagePath);	50✔
257	const packageScripts = _.reduce(	50✔
258	this.configuration.packagerOptions.scripts \|\| [],	50✔
259	(__, script, index) => {	50✔
260	__[`script${index}`] = script;	×
261	return __;	×
262	},	50✔
263	{}	50✔
264	);	50✔
265		50✔
266	// Determine and create packager	50✔
267	return BbPromise.try(() => Packagers.get.call(this, this.configuration.packager)).then(packager => {	50✔
268	// Fetch needed original package.json sections	46✔
269	const sectionNames = packager.copyPackageSectionNames(this.configuration.packagerOptions);	46✔
270	const packageJson = this.serverless.utils.readFileSync(packageJsonPath);	46✔
271	const packageSections = _.pick(packageJson, sectionNames);	46✔
272	if (!_.isEmpty(packageSections)) {	46✔
273	if (this.log) {	4!
274	this.log.verbose(`Using package.json sections ${_.join(_.keys(packageSections), ', ')}`);	×
275	} else {	4✔
276	this.options.verbose &&	4✔
277	this.serverless.cli.log(`Using package.json sections ${_.join(_.keys(packageSections), ', ')}`);	4✔
278	}	4✔
279	}	4✔
280		46✔
281	// Get first level dependency graph	46✔
282	if (this.log) {	46!
UNCOV 283	this.log.verbose(`Fetch dependency graph from ${packageJsonPath}`);	×
284	} else {	46✔
285	this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);	46✔
286	}	46✔
287		46✔
288	return packager	46✔
289	.getProdDependencies(path.dirname(packageJsonPath), 1, this.configuration.packagerOptions)	46✔
290	.then(dependencyGraph => {	46✔
291	const problems = _.get(dependencyGraph, 'problems', []);	44✔
292	if (this.options.verbose && !_.isEmpty(problems)) {	44✔
293	if (this.log) {	2!
294	this.log.verbose(`Ignoring ${_.size(problems)} NPM errors:`);	×
295	} else {	2✔
296	this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);	2✔
297	}	2✔
298	_.forEach(problems, problem => {	2✔
299	if (this.log) {	4!
300	this.log.verbose(`=> ${problem}`);	×
301	} else {	4✔
302	this.serverless.cli.log(`=> ${problem}`);	4✔
303	}	4✔
304	});	2✔
305	}	2✔
306		44✔
307	// (1) Generate dependency composition	44✔
308	const compositeModules = _.uniq(	44✔
309	_.flatMap(stats.stats, compileStats => {	44✔
310	const externalModules = _.concat(	44✔
311	compileStats.externalModules,	44✔
312	_.map(packageForceIncludes, whitelistedPackage => ({	44✔
313	external: whitelistedPackage	6✔
314	}))	44✔
315	);	44✔
316	return getProdModules.call(	44✔
317	this,	44✔
318	externalModules,	44✔
319	packagePath,	44✔
320	nodeModulesRelativeDir,	44✔
321	dependencyGraph,	44✔
322	packageForceExcludes	44✔
323	);	44✔
324	})	44✔
325	);	44✔
326	removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);	44✔
327		44✔
328	if (_.isEmpty(compositeModules)) {	44✔
329	// The compiled code does not reference any external modules at all	2✔
330	if (this.log) {	2!
331	this.log('No external modules needed');	×
332	} else {	2✔
333	this.serverless.cli.log('No external modules needed');	2✔
334	}	2✔
335	return BbPromise.resolve();	2✔
336	}	2✔
337		40✔
338	// (1.a) Install all needed modules	40✔
339	const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');	40✔
340	const compositePackageJson = path.join(compositeModulePath, 'package.json');	40✔
341		40✔
342	// (1.a.1) Create a package.json	40✔
343	const compositePackage = _.defaults(	40✔
344	{	40✔
345	name: this.serverless.service.service,	40✔
346	version: '1.0.0',	40✔
347	description: `Packaged externals for ${this.serverless.service.service}`,	40✔
348	private: true,	40✔
349	scripts: packageScripts	40✔
350	},	40✔
351	packageSections	40✔
352	);	40✔
353	const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));	40✔
354	addModulesToPackageJson(compositeModules, compositePackage, relPath);	40✔
355	this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));	40✔
356		40✔
357	// (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades	40✔
358	const packagerOptions = this.configuration.packagerOptions \|\| {};	44!
359	const packageLockPath = path.join(	44✔
360	findWorkspaceRoot(path.dirname(packageJsonPath)) \|\| path.dirname(packageJsonPath),	44✔
361	packagerOptions.lockFile \|\| packager.lockfileName	44✔
362	);	44✔
363	let hasPackageLock = false;	44✔
364	return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))	44✔
365	.then(exists => {	44✔
366	if (exists) {	40✔
367	if (this.log) {	10!
UNCOV 368	this.log('Package lock found - Using locked versions');	×
369	} else {	10✔
370	this.serverless.cli.log('Package lock found - Using locked versions');	10✔
371	}	10✔
372	try {	10✔
373	let packageLockFile = this.serverless.utils.readFileSync(packageLockPath);	10✔
374	packageLockFile = packager.rebaseLockfile(relPath, packageLockFile);	10✔
375	if (_.isObject(packageLockFile)) {	10✔
376	packageLockFile = JSON.stringify(packageLockFile, null, 2);	6✔
377	}	6✔
378		6✔
379	this.serverless.utils.writeFileSync(	6✔
380	path.join(compositeModulePath, packager.lockfileName),	6✔
381	packageLockFile	6✔
382	);	6✔
383	hasPackageLock = true;	6✔
384	} catch (err) {	10✔
385	if (this.log) {	4!
386	this.log.warning(`Could not read lock file: ${err.message}`);	×
387	} else {	4✔
388	this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);	4✔
389	}	4✔
390	}	4✔
391	}	10✔
392	return BbPromise.resolve();	40✔
393	})	44✔
394	.then(() => {	44✔
395	const start = _.now();	40✔
396	if (this.log) {	40!
UNCOV 397	this.log('Packing external modules: ' + compositeModules.join(', '));	×
398	} else {	40✔
399	this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));	40✔
400	}	40✔
401		40✔
402	return packager.getPackagerVersion(compositeModulePath).then(version => {	40✔
403	return packager	40✔
404	.install(compositeModulePath, this.configuration.packagerOptions, version)	40✔
405	.then(() => {	40✔
406	if (this.log) {	38!
UNCOV 407	this.log.verbose(`Package took [${_.now() - start} ms]`);	×
408	} else {	38✔
409	this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`);	38✔
410	}	38✔
411	return null;	38✔
412	})	40✔
413	.return(stats.stats);	40✔
414	});	40✔
415	})	44✔
416	.mapSeries(compileStats => {	44✔
417	const modulePath = compileStats.outputPath;	38✔
418		38✔
419	// Create package.json	38✔
420	const modulePackageJson = path.join(modulePath, 'package.json');	38✔
421	const modulePackage = _.defaults(	38✔
422	{	38✔
423	name: this.serverless.service.service,	38✔
424	version: '1.0.0',	38✔
425	description: `Packaged externals for ${this.serverless.service.service}`,	38✔
426	private: true,	38✔
427	scripts: packageScripts,	38✔
428	dependencies: {}	38✔
429	},	38✔
430	packageSections	38✔
431	);	38✔
432	const prodModules = getProdModules.call(	38✔
433	this,	38✔
434	_.concat(	38✔
435	compileStats.externalModules,	38✔
436	_.map(packageForceIncludes, whitelistedPackage => ({	38✔
437	external: whitelistedPackage	6✔
438	}))	38✔
439	),	38✔
440	packagePath,	38✔
441	nodeModulesRelativeDir,	38✔
442	dependencyGraph,	38✔
443	packageForceExcludes	38✔
444	);	38✔
445	removeExcludedModules.call(this, prodModules, packageForceExcludes);	38✔
446	const relPath = path.relative(modulePath, path.dirname(packageJsonPath));	38✔
447	addModulesToPackageJson(prodModules, modulePackage, relPath);	38✔
448	this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));	38✔
449		38✔
450	// GOOGLE: Copy modules only if not google-cloud-functions	38✔
451	// GCF Auto installs the package json	38✔
452	if (isProviderGoogle(this.serverless)) {	38✔
453	return BbPromise.resolve();	2✔
454	}	2✔
455		36✔
456	const startCopy = _.now();	36✔
457	return BbPromise.try(() => {	36✔
458	// Only copy dependency modules if demanded by packager	36✔
459	if (packager.mustCopyModules) {	36✔
460	return BbPromise.fromCallback(callback =>	34✔
461	fse.copy(	34✔
462	path.join(compositeModulePath, 'node_modules'),	34✔
463	path.join(modulePath, 'node_modules'),	34✔
464	callback	34✔
465	)	34✔
466	);	34✔
467	}	34✔
468	return BbPromise.resolve();	2✔
469	})	36✔
470	.then(() =>	36✔
471	hasPackageLock	36✔
472	? BbPromise.fromCallback(callback =>	6✔
473	fse.copy(	6✔
474	path.join(compositeModulePath, packager.lockfileName),	6✔
475	path.join(modulePath, packager.lockfileName),	6✔
476	callback	6✔
477	)	6✔
478	)	6✔
479	: BbPromise.resolve()	30✔
480	)	36✔
481	.tap(() => {	36✔
482	if (this.log) {	36!
UNCOV 483	this.log.verbose(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`);	×
484	} else {	36✔
485	this.options.verbose &&	36✔
486	this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`);	36✔
487	}	36✔
488	})	36✔
489	.then(() => {	36✔
490	// Prune extraneous packages - removes not needed ones	36✔
491	const startPrune = _.now();	36✔
492	return packager.getPackagerVersion(modulePath).then(version => {	36✔
493	return packager.prune(modulePath, this.configuration.packagerOptions, version).tap(() => {	36✔
494	if (this.log) {	36!
UNCOV 495	this.log.verbose(`Prune: ${modulePath} [${_.now() - startPrune} ms]`);	×
496	} else {	36✔
497	this.options.verbose &&	36✔
498	this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`);	36✔
499	}	36✔
500	});	36✔
501	});	36✔
502	})	36✔
503	.then(() => {	36✔
504	// Prune extraneous packages - removes not needed ones	36✔
505	const startRunScripts = _.now();	36✔
506	return packager.runScripts(modulePath, _.keys(packageScripts)).tap(() => {	36✔
507	if (this.log) {	36!
UNCOV 508	this.log.verbose(`Run scripts: ${modulePath} [${_.now() - startRunScripts} ms]`);	×
509	} else {	36✔
510	this.options.verbose &&	36✔
511	this.serverless.cli.log(`Run scripts: ${modulePath} [${_.now() - startRunScripts} ms]`);	36✔
512	}	36✔
513	});	36✔
514	});	36✔
515	})	44✔
516	.return();	44✔
517	});	46✔
518	});	50✔
519	}	50✔
520	};	2✔

serverless-heaven / serverless-webpack / 9560614935

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous