build: pass directory instead of list of files to js2c.py

On Windows there is a limit to the length of commands, so there
will be an error once the lengths of the JS file names combined
exceed that limit. This patch modifies js2c.py so that
it now takes a --directory argument to glob for .js and
.mjs files in addition to the list of files passed directly.
We still pass the additional files we include from deps/
directly through the command line, as we only includes some of
them so we cannot simply glob, but those are limited so listing
them out should be fine.

Refs: https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation

PR-URL: #39069
Refs: #38971
Reviewed-By: Chengzhong Wu <[email protected]>
Reviewed-By: James M Snell <[email protected]>

Author: joyeecheung

node.gyp

@@ -27,251 +27,15 @@
     'node_lib_target_name%': 'libnode',
     'node_intermediate_lib_type%': 'static_library',
     'node_builtin_modules_path%': '',
+    # We list the deps/ files out instead of globbing them in js2c.py since we
+    # only include a subset of all the files under these directories.
+    # The lengths of their file names combined should not exceed the
+    # Windows command length limit or there would be an error.
+    # See https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation
     'library_files': [
-      'lib/internal/bootstrap/environment.js',
-      'lib/internal/bootstrap/loaders.js',
-      'lib/internal/bootstrap/node.js',
-      'lib/internal/bootstrap/pre_execution.js',
-      'lib/internal/bootstrap/switches/does_own_process_state.js',
-      'lib/internal/bootstrap/switches/does_not_own_process_state.js',
-      'lib/internal/bootstrap/switches/is_main_thread.js',
-      'lib/internal/bootstrap/switches/is_not_main_thread.js',
-      'lib/internal/per_context/primordials.js',
-      'lib/internal/per_context/domexception.js',
-      'lib/internal/per_context/messageport.js',
-      'lib/async_hooks.js',
-      'lib/assert.js',
-      'lib/assert/strict.js',
-      'lib/buffer.js',
-      'lib/child_process.js',
-      'lib/console.js',
-      'lib/constants.js',
-      'lib/crypto.js',
-      'lib/cluster.js',
-      'lib/diagnostics_channel.js',
-      'lib/dgram.js',
-      'lib/dns.js',
-      'lib/dns/promises.js',
-      'lib/domain.js',
-      'lib/events.js',
-      'lib/fs.js',
-      'lib/fs/promises.js',
-      'lib/http.js',
-      'lib/http2.js',
-      'lib/_http_agent.js',
-      'lib/_http_client.js',
-      'lib/_http_common.js',
-      'lib/_http_incoming.js',
-      'lib/_http_outgoing.js',
-      'lib/_http_server.js',
-      'lib/https.js',
-      'lib/inspector.js',
-      'lib/module.js',
-      'lib/net.js',
-      'lib/os.js',
-      'lib/path.js',
-      'lib/path/posix.js',
-      'lib/path/win32.js',
-      'lib/perf_hooks.js',
-      'lib/process.js',
-      'lib/punycode.js',
-      'lib/querystring.js',
-      'lib/readline.js',
-      'lib/repl.js',
-      'lib/stream.js',
-      'lib/stream/promises.js',
-      'lib/_stream_readable.js',
-      'lib/_stream_writable.js',
-      'lib/_stream_duplex.js',
-      'lib/_stream_transform.js',
-      'lib/_stream_passthrough.js',
-      'lib/_stream_wrap.js',
-      'lib/string_decoder.js',
-      'lib/sys.js',
-      'lib/timers/promises.js',
-      'lib/timers.js',
-      'lib/tls.js',
-      'lib/_tls_common.js',
-      'lib/_tls_wrap.js',
-      'lib/trace_events.js',
-      'lib/tty.js',
-      'lib/url.js',
-      'lib/util.js',
-      'lib/util/types.js',
-      'lib/v8.js',
-      'lib/vm.js',
-      'lib/wasi.js',
-      'lib/worker_threads.js',
-      'lib/zlib.js',
-      'lib/internal/abort_controller.js',
-      'lib/internal/assert.js',
-      'lib/internal/assert/assertion_error.js',
-      'lib/internal/assert/calltracker.js',
-      'lib/internal/async_hooks.js',
-      'lib/internal/blob.js',
-      'lib/internal/blocklist.js',
-      'lib/internal/buffer.js',
-      'lib/internal/cli_table.js',
-      'lib/internal/child_process.js',
-      'lib/internal/child_process/serialization.js',
-      'lib/internal/cluster/child.js',
-      'lib/internal/cluster/primary.js',
-      'lib/internal/cluster/round_robin_handle.js',
-      'lib/internal/cluster/shared_handle.js',
-      'lib/internal/cluster/utils.js',
-      'lib/internal/cluster/worker.js',
-      'lib/internal/console/constructor.js',
-      'lib/internal/console/global.js',
-      'lib/internal/crypto/aes.js',
-      'lib/internal/crypto/certificate.js',
-      'lib/internal/crypto/cipher.js',
-      'lib/internal/crypto/diffiehellman.js',
-      'lib/internal/crypto/dsa.js',
-      'lib/internal/crypto/ec.js',
-      'lib/internal/crypto/hash.js',
-      'lib/internal/crypto/hashnames.js',
-      'lib/internal/crypto/hkdf.js',
-      'lib/internal/crypto/keygen.js',
-      'lib/internal/crypto/keys.js',
-      'lib/internal/crypto/mac.js',
-      'lib/internal/crypto/pbkdf2.js',
-      'lib/internal/crypto/random.js',
-      'lib/internal/crypto/rsa.js',
-      'lib/internal/crypto/scrypt.js',
-      'lib/internal/crypto/sig.js',
-      'lib/internal/crypto/util.js',
-      'lib/internal/crypto/webcrypto.js',
-      'lib/internal/crypto/x509.js',
-      'lib/internal/constants.js',
-      'lib/internal/debugger/_inspect.js',
-      'lib/internal/debugger/inspect_client.js',
-      'lib/internal/debugger/inspect_repl.js',
-      'lib/internal/dgram.js',
-      'lib/internal/dns/promises.js',
-      'lib/internal/dns/utils.js',
-      'lib/internal/dtrace.js',
-      'lib/internal/encoding.js',
-      'lib/internal/errors.js',
-      'lib/internal/error_serdes.js',
-      'lib/internal/event_target.js',
-      'lib/internal/fixed_queue.js',
-      'lib/internal/freelist.js',
-      'lib/internal/freeze_intrinsics.js',
-      'lib/internal/fs/dir.js',
-      'lib/internal/fs/promises.js',
-      'lib/internal/fs/read_file_context.js',
-      'lib/internal/fs/rimraf.js',
-      'lib/internal/fs/streams.js',
-      'lib/internal/fs/sync_write_stream.js',
-      'lib/internal/fs/utils.js',
-      'lib/internal/fs/watchers.js',
-      'lib/internal/http.js',
-      'lib/internal/heap_utils.js',
-      'lib/internal/histogram.js',
-      'lib/internal/idna.js',
-      'lib/internal/inspector_async_hook.js',
-      'lib/internal/js_stream_socket.js',
-      'lib/internal/legacy/processbinding.js',
-      'lib/internal/linkedlist.js',
-      'lib/internal/main/check_syntax.js',
-      'lib/internal/main/eval_string.js',
-      'lib/internal/main/eval_stdin.js',
-      'lib/internal/main/inspect.js',
-      'lib/internal/main/print_help.js',
-      'lib/internal/main/prof_process.js',
-      'lib/internal/main/repl.js',
-      'lib/internal/main/run_main_module.js',
-      'lib/internal/main/worker_thread.js',
-      'lib/internal/modules/run_main.js',
-      'lib/internal/modules/package_json_reader.js',
-      'lib/internal/modules/cjs/helpers.js',
-      'lib/internal/modules/cjs/loader.js',
-      'lib/internal/modules/esm/loader.js',
-      'lib/internal/modules/esm/create_dynamic_module.js',
-      'lib/internal/modules/esm/get_format.js',
-      'lib/internal/modules/esm/get_source.js',
-      'lib/internal/modules/esm/module_job.js',
-      'lib/internal/modules/esm/module_map.js',
-      'lib/internal/modules/esm/resolve.js',
-      'lib/internal/modules/esm/transform_source.js',
-      'lib/internal/modules/esm/translators.js',
-      'lib/internal/net.js',
-      'lib/internal/options.js',
-      'lib/internal/perf/perf.js',
-      'lib/internal/perf/nodetiming.js',
-      'lib/internal/perf/usertiming.js',
-      'lib/internal/perf/observe.js',
-      'lib/internal/perf/event_loop_delay.js',
-      'lib/internal/perf/event_loop_utilization.js',
-      'lib/internal/perf/timerify.js',
-      'lib/internal/policy/manifest.js',
-      'lib/internal/policy/sri.js',
-      'lib/internal/priority_queue.js',
-      'lib/internal/process/esm_loader.js',
-      'lib/internal/process/execution.js',
-      'lib/internal/process/per_thread.js',
-      'lib/internal/process/policy.js',
-      'lib/internal/process/promises.js',
-      'lib/internal/process/warning.js',
-      'lib/internal/process/worker_thread_only.js',
-      'lib/internal/process/report.js',
-      'lib/internal/process/signal.js',
-      'lib/internal/process/task_queues.js',
-      'lib/internal/querystring.js',
-      'lib/internal/readline/callbacks.js',
-      'lib/internal/readline/emitKeypressEvents.js',
-      'lib/internal/readline/utils.js',
-      'lib/internal/repl.js',
-      'lib/internal/repl/await.js',
-      'lib/internal/repl/history.js',
-      'lib/internal/repl/utils.js',
-      'lib/internal/socketaddress.js',
-      'lib/internal/socket_list.js',
-      'lib/internal/source_map/prepare_stack_trace.js',
-      'lib/internal/source_map/source_map.js',
-      'lib/internal/source_map/source_map_cache.js',
-      'lib/internal/test/binding.js',
-      'lib/internal/test/transfer.js',
-      'lib/internal/timers.js',
-      'lib/internal/tls.js',
-      'lib/internal/trace_events_async_hooks.js',
-      'lib/internal/tty.js',
-      'lib/internal/url.js',
-      'lib/internal/util.js',
-      'lib/internal/util/comparisons.js',
-      'lib/internal/util/debuglog.js',
-      'lib/internal/util/inspect.js',
-      'lib/internal/util/inspector.js',
-      'lib/internal/util/iterable_weak_map.js',
-      'lib/internal/util/types.js',
-      'lib/internal/http2/core.js',
-      'lib/internal/http2/compat.js',
-      'lib/internal/http2/util.js',
-      'lib/internal/v8_prof_polyfill.js',
-      'lib/internal/v8_prof_processor.js',
-      'lib/internal/validators.js',
-      'lib/internal/stream_base_commons.js',
-      'lib/internal/vm/module.js',
-      'lib/internal/worker.js',
-      'lib/internal/worker/io.js',
-      'lib/internal/worker/js_transferable.js',
-      'lib/internal/watchdog.js',
-      'lib/internal/streams/lazy_transform.js',
-      'lib/internal/streams/add-abort-signal.js',
-      'lib/internal/streams/buffer_list.js',
-      'lib/internal/streams/duplexpair.js',
-      'lib/internal/streams/from.js',
-      'lib/internal/streams/legacy.js',
-      'lib/internal/streams/readable.js',
-      'lib/internal/streams/writable.js',
-      'lib/internal/streams/duplex.js',
-      'lib/internal/streams/passthrough.js',
-      'lib/internal/streams/transform.js',
-      'lib/internal/streams/destroy.js',
-      'lib/internal/streams/state.js',
-      'lib/internal/streams/pipeline.js',
-      'lib/internal/streams/end-of-stream.js',
-      'lib/internal/streams/utils.js',
+      '<!@(python tools/search_files.py --ext js lib)',
+    ],
+    'deps_files': [
       'deps/v8/tools/splaytree.mjs',
       'deps/v8/tools/codemap.mjs',
       'deps/v8/tools/consarray.mjs',
@@ -794,6 +558,7 @@
         'deps/v8/include/v8.h',
         # javascript files to make for an even more pleasant IDE experience
         '<@(library_files)',
+        '<@(deps_files)',
         # node.gyp is added by default, common.gypi is added for change detection
         'common.gypi',
       ],
@@ -1047,14 +812,21 @@
             # Put the code first so it's a dependency and can be used for invocation.
             'tools/js2c.py',
             '<@(library_files)',
+            '<@(deps_files)',
             'config.gypi'
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
           ],
           'action': [
-            'python', '<@(_inputs)',
-            '--target', '<@(_outputs)',
+            'python',
+            'tools/js2c.py',
+            '--directory',
+            'lib',
+            '--target',
+            '<@(_outputs)',
+            'config.gypi',
+            '<@(deps_files)',
           ],
         },
       ],
@@ -1598,6 +1370,7 @@
           ],
           'sources': [
             '<@(library_files)',
+            '<@(deps_files)',
             'common.gypi',
           ],
           'direct_dependent_settings': {

tools/js2c.py

@@ -36,6 +36,7 @@
 import re
 import functools
 import codecs
+import utils
 
 def ReadFile(filename):
   if is_verbose:
@@ -204,12 +205,23 @@ def main():
     fromfile_prefix_chars='@'
   )
   parser.add_argument('--target', help='output file')
+  parser.add_argument(
+      '--directory',
+      default=None,
+      help='input file directory')
   parser.add_argument('--verbose', action='store_true', help='output file')
   parser.add_argument('sources', nargs='*', help='input files')
   options = parser.parse_args()
   global is_verbose
   is_verbose = options.verbose
-  source_files = functools.reduce(SourceFileByExt, options.sources, {})
+  sources = options.sources
+  if options.directory is not None:
+    js_files = utils.SearchFiles(options.directory, 'js')
+    mjs_files = utils.SearchFiles(options.directory, 'mjs')
+    sources = js_files + mjs_files + options.sources
+
+  source_files = functools.reduce(SourceFileByExt, sources, {})
+
   # Should have exactly 3 types: `.js`, `.mjs` and `.gypi`
   assert len(source_files) == 3
   # Currently config.gypi is the only `.gypi` file allowed

tools/search_files.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+"""
+This is a utility for recursively searching files under
+a specified directory
+"""
+
+import argparse
+import utils
+
+def main():
+  parser = argparse.ArgumentParser(
+    description='Search files with a specific extension under a directory',
+    fromfile_prefix_chars='@'
+  )
+  parser.add_argument('--ext', required=True, help='extension to search for')
+  parser.add_argument('directory', help='input directory')
+  options = parser.parse_args()
+  print('\n'.join(utils.SearchFiles(options.directory, options.ext)))
+
+if __name__ == "__main__":
+  main()

tools/utils.py

@@ -26,8 +26,10 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+import glob
 import platform
 import re
+import sys
 
 
 # Reads a .list file into an array of strings
@@ -106,3 +108,10 @@ def GuessArchitecture():
 
 def IsWindows():
   return GuessOS() == 'win32'
+
+
+def SearchFiles(dir, ext):
+  list = glob.glob(dir+ '/**/*.' + ext, recursive=True)
+  if sys.platform == 'win32':
+    list = [ x.replace('\\', '/')for x in list]
+  return list