Default metro.config.js should be configured to support symbol links to packages

[ryantrem]

Describe the Feature

npm install <path to local package> creates a symbolic link to the package directory. This is useful for monorepos as well as iterating on a package in the context of the consuming app. Unfortunately, the Metro bundler does not support this out of the box (see facebook/metro#1). It is possible though to configure the Metro bundler (via metro.config.js) to make it work with symbolically linked packages. It would be extremely helpful if react-native init MyApp provided a metro.config.js that made symbolically linked packages work by default.

Possible Implementations

There are many work arounds proposed in facebook/metro#1. All of them had problems for me. My solution (used in multiple projects) is to modify metro.config.js to manually follow symbolic links:

/**
 * Metro configuration for React Native
 * https://github.com/facebook/react-native
 *
 * @format
 */

const path = require('path');
const fs = require('fs');
const appendExclusions = require('metro-config/src/defaults/blacklist');

// Escape function taken from the MDN documentation: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#Escaping
function escapeRegExp(string) {
  return string.replace(/[.*+\-?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
}

// NOTE: The Metro bundler does not support symlinks (see https://github.com/facebook/metro/issues/1), which NPM uses for local packages.
//       To work around this, we supplement the logic to follow symbolic links.

// Create a mapping of package ids to linked directories.
function processModuleSymLinks() {
  const nodeModulesPath = path.resolve(__dirname, 'node_modules');
  let moduleMappings = {};
  let moduleExclusions = [];

  function findPackageDirs(directory) {
    fs.readdirSync(directory).forEach(item => {
      const itemPath = path.resolve(directory, item);
      const itemStat = fs.lstatSync(itemPath);
      if (itemStat.isSymbolicLink()) {
        let linkPath = fs.readlinkSync(itemPath);
        // Sym links are relative in Unix, absolute in Windows.
        if (!path.isAbsolute(linkPath)) {
          linkPath = path.resolve(directory, linkPath);
        }
        const linkStat = fs.statSync(linkPath);
        if (linkStat.isDirectory()) {
          const packagePath = path.resolve(linkPath, "package.json");
          if (fs.existsSync(packagePath)) {
            const packageId = path.relative(nodeModulesPath, itemPath);
            moduleMappings[packageId] = linkPath;

            const packageInfoData = fs.readFileSync(packagePath);
            const packageInfo = JSON.parse(packageInfoData);

            const dependencies = packageInfo.dependencies ? Object.keys(packageInfo.dependencies) : [];
            const peerDependencies = packageInfo.peerDependencies ? Object.keys(packageInfo.peerDependencies) : [];
            const devDependencies = packageInfo.devDependencies ? Object.keys(packageInfo.devDependencies) : [];

            // Exclude dependencies that appear in devDependencies or peerDependencies but not in dependencies. Otherwise,
            // the metro bundler will package those devDependencies/peerDependencies as unintended copies.
            for (const devDependency of devDependencies.concat(peerDependencies).filter(dependency => !dependencies.includes(dependency))) {
              moduleExclusions.push(new RegExp(escapeRegExp(path.join(linkPath, "node_modules", devDependency)) + "\/.*"));
            }
          }
        }
      } else if (itemStat.isDirectory()) {
        findPackageDirs(itemPath);
      }
    });
  }

  findPackageDirs(nodeModulesPath);

  return [moduleMappings, moduleExclusions];
}

const [moduleMappings, moduleExclusions] = processModuleSymLinks();
console.log("Mapping the following sym linked packages:");
console.log(moduleMappings);

module.exports = {
  transformer: {
    getTransformOptions: async () => ({
      transform: {
        experimentalImportSupport: false,
        inlineRequires: false,
      },
    }),
  },

  resolver: {
    // Register an "extra modules proxy" for resolving modules outside of the normal resolution logic.
    extraNodeModules: new Proxy(
        // Provide the set of known local package mappings.
        moduleMappings,
        {
            // Provide a mapper function, which uses the above mappings for associated package ids,
            // otherwise fall back to the standard behavior and just look in the node_modules directory.
            get: (target, name) => name in target ? target[name] : path.join(__dirname, `node_modules/${name}`),
        },
    ),
    blacklistRE: appendExclusions(moduleExclusions),
  },

  projectRoot: path.resolve(__dirname),

  // Also additionally watch all the mapped local directories for changes to support live updates.
  watchFolders: Object.values(moduleMappings),
};

Related Issues

facebook/metro#1

[Braincompiler]

Thanks for this, it helps us to manage a bigger codebase where we use a lot of symlinks caused by multirepos.

But we had a problem the symlinks where not considered even with your metro.config.js. The problem is, you use fs.lstatSync(linkPath); for the already resolved symlink directory, this leads to the problem linkStat.isDirectory() returned false. I found out, you should use lstat only for the symbolic not for the resolved target anymore (nodejs/node#25342). After changing it to fs.statSync(linkPath); it was working as expected.

[ryantrem]

Thanks @Braincompiler for sharing back your findings! I read the link you shared and it certainly sounds like fs.lstatSync(linkPath) should not work, but it does work for me. Can you describe your specific scenario a bit more? Like what your directory structure looks like under node_modules and where in that structure the sym link is?

Also, I tested it out with fs.statSync(linkPath) and it also worked as expected for me. Since based on the links you shared this seems more likely correct, and since it also fixed the issue you were running into, I am updating my original post to reflect this change. I'd still love to better understand which specific scenarios are broken with the old way though!

[Braincompiler]

We debugged the metro bundler start and found out that linkStat.isDirectory() returned false which was not possible cause the already resolved symbolic link was a directory. Then we searched for that error in case of symbolic links and we found the linked issue. After using statSync rather than 'lstatSync' it worked as expcted.

Our directory structure looks like:

node_modules/
  @vendor-name
    module1 -> symbolic link
    module2 -> symbolic link
    module3 -> symbolic link

The only problem we now have: In case we change something in one of the linked modules, the app reloads but is not rebundled. Thus we have to restart the bundler yarn start to see our changes. Thats very annoying but we haven't found a working solution for that. The watchFolders: Object.values(moduleMappings), seems to work as the app reloads, but its not rebundled. For sake of completeness: We use Typescript, so the modules will also build after changing. We notice a double reload of the app, however it's not rebundled.

[ryantrem]

That's really interesting, I don't think we see any of those issues. We have exactly the same directory structure as you describe above and no issues with the link resolution even with lstatSync. When you say the app reloads, but it is not rebundled, does that effectively mean fast refresh does not work for you? This is also something that works as expected for us.

[Braincompiler]

Do you use Mac Books? We have currently Mid 2015 Mac Books, could this be a problem? I don't think so.

The refresh works (at least the app is reloading (fast)) but the changes we made in one of the dependent modules are not there. We have to exit the bundler and start it again. But this takes a lot of time, obviously.

[ryantrem]

Yes we use both Mac and Windows. We did have an issue at one point kind of like what you are describing, but for us if we saved the top level App.js it would refresh as expected. However this stopped reproing before we had a chance to investigate so we assumed it was fixed by a React Native update.

[Braincompiler]

Since you said you saved the "App.js", means you don't use Typescript? We use Typescript (formerly the cli generated typescript project with babel, now we use a custom build step using tsc so we can use tsconfig and better typechecking support). So maybe the problem is indicated due to the extra transpile step? But the changes were never built in even after manual refresh (⌘+R).

[ryantrem]

We use TypeScript + Babel, I just said App.js to keep things simple. In reality it is App.tsx for us. What version of React Native and the CLI tools are you using? I’m not sure what other tools come into play here... node, watchman, other?

[Braincompiler]

We use always the latest versions for all the tools and clis you named. Currently we don't use symlinks anymore because the problems with the bundler were too hard to work with. But hopefully there will be a solution for this provided by the metro bundler itself. I cannot imagine Facebook is not using symlinks or similar in there big dev environments.

[Merlier]

Describe the Feature

npm install <path to local package> creates a symbolic link to the package directory. This is useful for monorepos as well as iterating on a package in the context of the consuming app. Unfortunately, the Metro bundler does not support this out of the box (see facebook/metro#1). It is possible though to configure the Metro bundler (via metro.config.js) to make it work with symbolically linked packages. It would be extremely helpful if react-native init MyApp provided a metro.config.js that made symbolically linked packages work by default.

Possible Implementations

There are many work arounds proposed in facebook/metro#1. All of them had problems for me. My solution (used in multiple projects) is to modify metro.config.js to manually follow symbolic links:

/**
 * Metro configuration for React Native
 * https://github.com/facebook/react-native
 *
 * @format
 */

const path = require('path');
const fs = require('fs');

// NOTE: The Metro bundler does not support symlinks (see https://github.com/facebook/metro/issues/1), which NPM uses for local packages.
//       To work around this, we supplement the logic to follow symbolic links.

// Create a mapping of package ids to linked directories.
function getModuleMappings() {
  const nodeModulesPath = path.resolve(__dirname, 'node_modules');
  let moduleMappings = {};

  function findPackageDirs(directory) {
    fs.readdirSync(directory).forEach(item => {
      const itemPath = path.resolve(directory, item);
      const itemStat = fs.lstatSync(itemPath);
      if (itemStat.isSymbolicLink()) {
        let linkPath = fs.readlinkSync(itemPath);
        // Sym links are relative in Unix, absolute in Windows.
        if (!path.isAbsolute(linkPath)) {
          linkPath = path.resolve(directory, linkPath);
        }
        const linkStat = fs.statSync(linkPath);
        if (linkStat.isDirectory()) {
          const packagePath = path.resolve(linkPath, "package.json");
          if (fs.existsSync(packagePath)) {
            const packageId = path.relative(nodeModulesPath, itemPath);
            moduleMappings[packageId] = linkPath;
          }
        }
      } else if (itemStat.isDirectory()) {
        findPackageDirs(itemPath);
      }
    });
  }

  findPackageDirs(nodeModulesPath);

  return moduleMappings;
}

const moduleMappings = getModuleMappings();
console.log("Mapping the following sym linked packages:");
console.log(moduleMappings);

module.exports = {
  transformer: {
    getTransformOptions: async () => ({
      transform: {
        experimentalImportSupport: false,
        inlineRequires: false,
      },
    }),
  },

  resolver: {
    // Register an "extra modules proxy" for resolving modules outside of the normal resolution logic.
    extraNodeModules: new Proxy(
        // Provide the set of known local package mappings.
        moduleMappings,
        {
            // Provide a mapper function, which uses the above mappings for associated package ids,
            // otherwise fall back to the standard behavior and just look in the node_modules directory.
            get: (target, name) => name in target ? target[name] : path.join(__dirname, `node_modules/${name}`),
        },
    ),
  },

  projectRoot: path.resolve(__dirname),

  // Also additionally watch all the mapped local directories for changes to support live updates.
  watchFolders: Object.values(moduleMappings),
};

Related Issues

facebook/metro#1

Thanks for your solution 👍

[ryantrem]

We found that in cases where you sym link to a package that has devDependencies, those dependencies are installed under that package's node_modules, and those get picked up (as a copy) by the Metro bundler. In a non-symlinked package, this wouldn't be an issue because because devDependencies would not be installed with the package. To fix this, we added some additional logic that adds any devDependencies of sym linked packages to the path exclusion list. I've updated my original post to include this fix.

[carloseustaquio]

Thanks for this @ryantrem ! I've been searching for a way to do this for many weeks.
Unfortulately, I couldn't manage this to work yet. When I do the linking and run the metro I get the following log:

$ react-native start --config dev.metro.config.js
Mapping the following sym linked packages:
{ '@my-company/my-package':
   '/home/myuser/.config/yarn/link/@my-company/my-package' }

And in fact, it's looking for files in the linked path:

Looking for JS files in
   /home/carlos/code/vizir/my-app
   /home/carlos/.config/yarn/link/@my-company/my-package

It compiles ok, but when I try to open the app I get the following error:

  error: bundling failed: Error: Unable to resolve module `@my-company/my-package/dist/navigation/app_stack.native` from `src/index.native.tsx`: @my-company/my-package/dist/navigation/app_stack.native could not be found within the project or in these directories:
  /home/carlos/.config/yarn/link/@my-company/my-package/dist/navigation

When open the symlink path, the package is there.
I'm kind of a beginner in this metro configuration thing, and I'm not really shure about how I should try to debug this.
ps: @my-company/my-package has dependencies, devDependencies and peerDependencies

[ryantrem]

It looks like you are trying to use this with yarn link? I've never tried that, though I would expect it to work. In my case, I have sym links only through npm install of a local package.

What is app_stack.native? Is it some kind of embedded resource? If so, we've had problems with resource loading from sym linked modules using this approach, so happy to share what we learn when we finish our in progress investigation around this.

[kopax-polyconseil]

Hi @ryantrem and thanks for sharing your solution. I've been trying to link modules with msw, and haul, now I am trying your ultimate solution which is the closest to the desired result.

One thing that still doesn't work is the hot reload, despite that I reload the app with r, I keep having the old source being rendered.

This forced me to do the following in order to get the updated sources:

• force app quit
• restart metro process
• start app

I've tried to update the configuration without success:

  watchFolders: [
    ...Object.values(moduleMappings),
+    join(__dirname, 'node_modules', '@pass-culture', 'id-check', 'src'),
  ],

Do you have a clue why I can't get the hot reload on that folder?

This is what I have in watchFolders metro option:

['/home/dka/.config/yarn/link/@pass-culture/id-check']

This is what I have when I run watchman watch-list:

watchman watch-list
{
    "version": "20210124.162314.0",
    "roots": [
        "/home/dka/workspace/github.com/pass-culture/id-check-front/packages/id-check",
        "/home/dka/workspace/github.com/pass-culture/pass-culture-app-native"
    ]
}

Thanks a lot for this solution, I wonder why this is not the default if it can work well.

[ryantrem]

@kopax-polyconseil that sounds similar to the issue @Braincompiler was hitting. I've not run into this problem before, so I'm not sure why others do. You can check out our OSS project where we use this configuration and see if it works for you there, and if so, maybe you can find the difference between your repo and ours. https://github.com/babylonjs/babylonreactnative

[kopax-polyconseil]

Thanks for sharing the link, I had a look and it look equal. That's our metro.config.js. And yes we also do use TypeScript so it must be related.

[mikehardy]

This is magic! Thank you @ryantrem - was able to integrate this immediately into an example app from npx react-native init ... where I was locally testing a module I'm developing, "Just Worked(tm)"

[ryantrem]

I found that with NPM 7+, if you npm install in the module directory that is being linked, it automatically installs peerDependencies (default behavior for NPM 7+), and so those peerDependencies get erroneously bundled with the consuming app. This can be fixed by filtering out peerDependencies just like with devDependencies. I updated my solution to include this.

[AliRezaBeigy]

New metro.config.js to support metro-config v0.66.2

const path = require('path');
const exclusionList = require('metro-config/src/defaults/exclusionList');
const escape = require('escape-string-regexp');
const pak = require('../package.json');

const root = path.resolve(__dirname, '..');

const modules = Object.keys({
  ...pak.peerDependencies,
});

module.exports = {
  projectRoot: __dirname,
  watchFolders: [root],
  resolver: {
    blockList: exclusionList(
      modules.map(
        m => new RegExp(`^${escape(path.join(root, 'node_modules', m))}\\/.*$`),
      ),
    ),

    extraNodeModules: modules.reduce((acc, name) => {
      acc[name] = path.join(__dirname, 'node_modules', name);
      return acc;
    }, {}),
  },
  transformer: {
    getTransformOptions: async () => ({
      transform: {
        experimentalImportSupport: false,
        inlineRequires: true,
      },
    }),
  },
};

[Sup3r-Us3r]

@ryantrem Your solution works for me, but I faced an Invariant Violation problem, in the lib I'm building it has some dependencies like:

@react-native-community/slider
react-native-linear-gradient

These dependencies are also present in the application that will use this lib, so the error occurs, for example:

Invariant Violation: Tried to register two views with the same name BVLinearGradient

If I generate a build of the lib, and send it to NPM and install it in the app, it works normally, is there any solution so that I can use these libs in the development environment?

I tried to replace:

const appendExclusions = require('metro-config/src/defaults/blacklist');

per:

const exclusionList = require('metro-config/src/defaults/exclusionList');
const escape = require('escape-string-regexp');

And I added this in the resolver:

blockList: exclusionList(
  ['@react-native-community/slider', 'react-native-linear-gradient'].map(
    m => new RegExp(`^${escape(path.join(path.resolve(__dirname), 'node_modules/MY_LIB_HERE/node_modules', m))}\\/.*$`),
  ),
),

I thought that this would block these packages from my lib, thus keeping the ones that exist in the app, but it didn't work.

[github-actions]

There hasn't been any activity on this issue in the past 3 months, so it has been marked as stale and it will be closed automatically if no further activity occurs in the next 7 days.