Skip to content

Commit

Permalink
Add support for parsing submodules
Browse files Browse the repository at this point in the history
  • Loading branch information
pswaminathan committed May 22, 2022
1 parent daf209e commit 385747c
Show file tree
Hide file tree
Showing 7 changed files with 453 additions and 47 deletions.
42 changes: 42 additions & 0 deletions spm/private/modulemap_parser/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ bzl_library(
srcs = ["collect_module.bzl"],
visibility = ["//spm:__subpackages__"],
deps = [
":collect_module_attribute",
":collect_module_members",
":collection_results",
":declarations",
Expand All @@ -70,10 +71,51 @@ bzl_library(
],
)

bzl_library(
name = "collect_module_attribute",
srcs = ["collect_module_attribute.bzl"],
visibility = ["//spm:__subpackages__"],
deps = [
":collection_results",
":tokens",
],
)

bzl_library(
name = "collect_module_members",
srcs = ["collect_module_members.bzl"],
visibility = ["//spm:__subpackages__"],
deps = [
":collect_export_declaration",
":collect_header_declaration",
":collect_link_declaration",
":collect_submodule",
":collect_umbrella_dir_declaration",
":collection_results",
":errors",
":tokens",
"@bazel_skylib//lib:sets",
],
)

bzl_library(
name = "collect_submodule",
srcs = ["collect_submodule.bzl"],
visibility = ["//spm:__subpackages__"],
deps = [
":collect_module_attribute",
":collect_submodule_members",
":collection_results",
":declarations",
":errors",
":tokens",
],
)

bzl_library(
name = "collect_submodule_members",
srcs = ["collect_submodule_members.bzl"],
visibility = ["//spm:__subpackages__"],
deps = [
":collect_export_declaration",
":collect_header_declaration",
Expand Down
50 changes: 5 additions & 45 deletions spm/private/modulemap_parser/collect_module.bzl
Original file line number Diff line number Diff line change
@@ -1,51 +1,15 @@
"""Defintion for collect_module."""

load(":collect_module_attribute.bzl", "collect_module_attribute")
load(":collect_module_members.bzl", "collect_module_members")
load(":collection_results.bzl", "collection_results")
load(":declarations.bzl", "declarations")
load(":errors.bzl", "errors")
load(":tokens.bzl", "tokens", rws = "reserved_words", tts = "token_types")

# MARK: - Attribute Collection

def _collect_attribute(parsed_tokens):
"""Collect a module attribute.
Spec: https://clang.llvm.org/docs/Modules.html#attributes
Syntax:
attributes:
attribute attributesopt
attribute:
'[' identifier ']'
Args:
parsed_tokens: A `list` of tokens.
Returns:
A `tuple` where the first item is the collection result and the second is an
error `struct` as returned from errors.create().
"""
tlen = len(parsed_tokens)

_open_token, err = tokens.get_as(parsed_tokens, 0, tts.square_bracket_open, count = tlen)
if err != None:
return None, err

attrib_token, err = tokens.get_as(parsed_tokens, 1, tts.identifier, count = tlen)
if err != None:
return None, err

_open_token, err = tokens.get_as(parsed_tokens, 2, tts.square_bracket_close, count = tlen)
if err != None:
return None, err

return collection_results.new([attrib_token.value], 3), None

# MARK: - Module Collection

def collect_module(parsed_tokens, is_submodule = False, prefix_tokens = []):
def collect_module(parsed_tokens, prefix_tokens = []):
"""Collect a module declaration.
Spec: https://clang.llvm.org/docs/Modules.html#module-declaration
Expand All @@ -55,14 +19,12 @@ def collect_module(parsed_tokens, is_submodule = False, prefix_tokens = []):
Args:
parsed_tokens: A `list` of tokens.
is_submodule: A `bool` that designates whether the module is a child of another module.
prefix_tokens: A `list` of tokens that have already been collected, but not applied.
Returns:
A `tuple` where the first item is the collection result and the second is an
error `struct` as returned from errors.create().
"""
explicit = False
framework = False
attributes = []
members = []
Expand All @@ -73,9 +35,7 @@ def collect_module(parsed_tokens, is_submodule = False, prefix_tokens = []):
# Process the prefix tokens
for token in prefix_tokens:
if token.type == tts.reserved and token.value == rws.explicit:
if not is_submodule:
return None, errors.new("The explicit qualifier can only exist on submodules.")
explicit = True
return None, errors.new("The explicit qualifier can only exist on submodules.")

elif token.type == tts.reserved and token.value == rws.framework:
framework = True
Expand Down Expand Up @@ -122,7 +82,7 @@ def collect_module(parsed_tokens, is_submodule = False, prefix_tokens = []):
break

elif tokens.is_a(token, tts.square_bracket_open):
collect_result, err = _collect_attribute(parsed_tokens[idx:])
collect_result, err = collect_module_attribute(parsed_tokens[idx:])
if err != None:
return None, err
attributes.extend(collect_result.declarations)
Expand All @@ -139,7 +99,7 @@ def collect_module(parsed_tokens, is_submodule = False, prefix_tokens = []):
# Create the declaration
decl = declarations.module(
module_id = module_id_token.value,
explicit = explicit,
explicit = False,
framework = framework,
attributes = attributes,
members = members,
Expand Down
41 changes: 41 additions & 0 deletions spm/private/modulemap_parser/collect_module_attribute.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
"""Defintion for collect_module_attribute."""

load(":collection_results.bzl", "collection_results")
load(":tokens.bzl", "tokens", tts = "token_types")

# MARK: - Attribute Collection

def collect_module_attribute(parsed_tokens):
"""Collect a module attribute.
Spec: https://clang.llvm.org/docs/Modules.html#attributes
Syntax:
attributes:
attribute attributesopt
attribute:
'[' identifier ']'
Args:
parsed_tokens: A `list` of tokens.
Returns:
A `tuple` where the first item is the collection result and the second is an
error `struct` as returned from errors.create().
"""
tlen = len(parsed_tokens)

_open_token, err = tokens.get_as(parsed_tokens, 0, tts.square_bracket_open, count = tlen)
if err != None:
return None, err

attrib_token, err = tokens.get_as(parsed_tokens, 1, tts.identifier, count = tlen)
if err != None:
return None, err

_open_token, err = tokens.get_as(parsed_tokens, 2, tts.square_bracket_close, count = tlen)
if err != None:
return None, err

return collection_results.new([attrib_token.value], 3), None
17 changes: 15 additions & 2 deletions spm/private/modulemap_parser/collect_module_members.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ load("@bazel_skylib//lib:sets.bzl", "sets")
load(":collect_export_declaration.bzl", "collect_export_declaration")
load(":collect_header_declaration.bzl", "collect_header_declaration")
load(":collect_link_declaration.bzl", "collect_link_declaration")
load(":collect_submodule.bzl", "collect_submodule")
load(":collect_umbrella_dir_declaration.bzl", "collect_umbrella_dir_declaration")
load(":collection_results.bzl", "collection_results")
load(":declarations.bzl", "declaration_types")
load(":errors.bzl", "errors")
load(":tokens.bzl", "tokens", rws = "reserved_words", tts = "token_types")

Expand Down Expand Up @@ -37,6 +39,7 @@ def collect_module_members(parsed_tokens):
skip_ahead = 0
collect_result = None
prefix_tokens = []
umbrella_decl = None
for idx in range(consumed_count, tlen - consumed_count):
consumed_count += 1
if skip_ahead > 0:
Expand All @@ -63,7 +66,7 @@ def collect_module_members(parsed_tokens):
elif tokens.is_a(token, tts.newline):
if len(prefix_tokens) > 0:
return None, errors.new(
"Unexpected prefix tokens found before end of line. tokens: %s" % (prefix_tokens),
"Unexpected prefix tokens found encountering newline before end of line. tokens: %s" % (prefix_tokens),
)

elif tokens.is_a(token, tts.reserved, rws.umbrella):
Expand All @@ -80,13 +83,19 @@ def collect_module_members(parsed_tokens):
else:
if len(prefix_tokens) > 0:
return None, errors.new(
"Unexpected prefix tokens found before end of line. tokens: %" %
"Unexpected prefix tokens found encountering umbrella dir before end of line. tokens: %" %
(prefix_tokens),
)
collect_result, err = collect_umbrella_dir_declaration(parsed_tokens[idx:])
if err == None and len(collect_result.declarations) == 1:
umbrella_decl = collect_result.declarations[0]

elif tokens.is_a(token, tts.reserved, rws.header):
collect_result, err = collect_header_declaration(parsed_tokens[idx:], prefix_tokens)
if (err == None and
len(collect_result.declarations) == 1 and
collect_result.declarations[0].decl_type == declaration_types.umbrella_header):
umbrella_decl = collect_result.declarations[0]
prefix_tokens = []

elif tokens.is_a(token, tts.reserved, rws.export):
Expand All @@ -95,6 +104,10 @@ def collect_module_members(parsed_tokens):
elif tokens.is_a(token, tts.reserved, rws.link):
collect_result, err = collect_link_declaration(parsed_tokens[idx:])

elif tokens.is_a(token, tts.reserved, rws.module):
collect_result, err = collect_submodule(parsed_tokens[idx:], prefix_tokens = prefix_tokens, umbrella_decl = umbrella_decl)
prefix_tokens = []

elif tokens.is_a(token, tts.reserved) and sets.contains(_unsupported_module_members, token.value):
return None, errors.new("Unsupported module member token. token: %s" % (token))

Expand Down
121 changes: 121 additions & 0 deletions spm/private/modulemap_parser/collect_submodule.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
"""Defintion for collect_submodule."""

load(":collect_module_attribute.bzl", "collect_module_attribute")
load(":collect_submodule_members.bzl", "collect_submodule_members")
load(":collection_results.bzl", "collection_results")
load(":declarations.bzl", "declarations")
load(":errors.bzl", "errors")
load(":tokens.bzl", "tokens", ops = "operators", rws = "reserved_words", tts = "token_types")

# MARK: - Module Collection

def collect_submodule(parsed_tokens, prefix_tokens = [], umbrella_decl = None):
"""Collect a submodule declaration.
Spec: https://clang.llvm.org/docs/Modules.html#submodule-declaration
Syntax:
explicitopt frameworkopt module module-id attributesopt '{' module-member* '}'
Args:
parsed_tokens: A `list` of tokens.
prefix_tokens: A `list` of tokens that have already been collected, but not applied.
umbrella_decl: A `declaration` of type `umbrella`, in the case of an inferred declaration.
Returns:
A `tuple` where the first item is the collection result and the second is an
error `struct` as returned from errors.create().
"""
explicit = False
framework = False
attributes = []
members = []
consumed_count = 0

tlen = len(parsed_tokens)

# Process the prefix tokens
for token in prefix_tokens:
if token.type == tts.reserved and token.value == rws.explicit:
explicit = True

elif token.type == tts.reserved and token.value == rws.framework:
framework = True

else:
return None, errors.new(
"Unexpected prefix token collecting module declaration. token: %s" % (token),
)

_module_token, err = tokens.get_as(parsed_tokens, 0, tts.reserved, rws.module, count = tlen)
if err != None:
return None, err
consumed_count += 1

module_id_token, err = tokens.get_as(parsed_tokens, 1, tts.identifier, count = tlen)
if err == None:
module_id = module_id_token.value
else:
if umbrella_decl == None:
return None, err

# A submodule without its next token as an identifier may be an inferred submodule.
# Such a submodule gets its members from the umbrella declaration provided in the
# parent module's members.
_, i_err = tokens.get_as(parsed_tokens, 1, tts.operator, ops.asterisk, count = tlen)
if i_err != None:
return None, i_err
module_id = umbrella_decl.path

consumed_count += 1

# Collect the attributes and module members
skip_ahead = 0
collect_result = None
for idx in range(consumed_count, tlen - consumed_count):
consumed_count += 1
if skip_ahead > 0:
skip_ahead -= 1
continue

collect_result = None
err = None

# Get next token
token, err = tokens.get(parsed_tokens, idx, count = tlen)
if err != None:
return None, err

# Process the token
if tokens.is_a(token, tts.curly_bracket_open):
collect_result, err = collect_submodule_members(parsed_tokens[idx:])
if err != None:
return None, err
members.extend(collect_result.declarations)
consumed_count += collect_result.count - 1
break

elif tokens.is_a(token, tts.square_bracket_open):
collect_result, err = collect_module_attribute(parsed_tokens[idx:])
if err != None:
return None, err
attributes.extend(collect_result.declarations)

else:
return None, errors.new(
"Unexpected token collecting attributes and module members. token: %s" % (token),
)

# Handle index advancement.
if collect_result:
skip_ahead = collect_result.count - 1

# Create the declaration
decl = declarations.module(
module_id = module_id,
explicit = explicit,
framework = framework,
attributes = attributes,
members = members,
)
return collection_results.new([decl], consumed_count), None
Loading

0 comments on commit 385747c

Please sign in to comment.