Consume bundles.config.extensions.json to amend TagWithExtensionSpec classes with extension version meta

bin/generate-validator-spec.php

@@ -12,10 +12,18 @@
     exit -1;
 }
 
-$specGenerator = new SpecGenerator();
-$destination   = !empty($argv[1]) ? $argv[1] : dirname(__DIR__) . '/src/Validator';
-$spec_url      = 'https://cdn.ampproject.org/v0/validator.json';
+$specGenerator      = new SpecGenerator();
+$destination        = !empty($argv[1]) ? $argv[1] : dirname(__DIR__) . '/src/Validator';
+$spec_url           = 'https://cdn.ampproject.org/v0/validator.json';
+$latest_release_url = 'https://api.github.com/repos/ampproject/amphtml/releases/latest';
+$bundle_config_url  = 'https://raw.githubusercontent.com/ampproject/amphtml/%s'
+                      . '/build-system/compile/bundles.config.extensions.json';
 
+/**
+ * Recursively remove an entire directory and all of its files/subdirectories.
+ *
+ * @param string $directory Directory to remove.
+ */
 function recursivelyRemoveDirectory($directory)
 {
     if (is_dir($directory)) {
@@ -37,24 +45,85 @@ function recursivelyRemoveDirectory($directory)
     }
 }
 
+/**
+ * Fetch JSON data from URL.
+ *
+ * @param string $url   JSON URL.
+ * @param bool   $cache Whether to cache response in temp directory.
+ *
+ * @return array Data.
+ * @throws RuntimeException         When the JSON file cannot be fetched.
+ * @throws RuntimeException         When the JSON data cannot be parsed.
+ * @throws UnexpectedValueException When the JSON data is not an array.
+ */
+function fetch_json($url, $cache = false)
+{
+    $stream_context = stream_context_create([
+        'http' => [
+            'user_agent' => 'amp-toolbox-php-generate-validator-spec/0.1',
+        ],
+    ]);
+
+    $json       = null;
+    $cache_file = sys_get_temp_dir() . '/amp-toolbox-php-' . md5($url);
+
+    if ($cache && file_exists($cache_file)) {
+        $json = file_get_contents($cache_file);
+    }
+
+    if ($json === null || $json === false) {
+        $json = file_get_contents($url, false, $stream_context);
+    }
+
+    if ($json === false) {
+        throw new RuntimeException('Failed to retrieve the JSON file at: ' . $url);
+    }
+
+    $data = json_decode($json, true);
+
+    if (json_last_error()) {
+        throw new RuntimeException('JSON parse error: ' . json_last_error_msg());
+    }
+
+    if (!is_array($data)) {
+        throw new UnexpectedValueException('Expected an associative array.');
+    }
+
+    if ($cache) {
+        file_put_contents($cache_file, $json);
+    }
+
+    return $data;
+}
+
 try {
+    $cache = ! empty($_ENV['CACHE_FETCHES']);
+
     echo "Recursively removing $destination";
     recursivelyRemoveDirectory($destination);
     echo "\n";
 
-    echo "Fetching $spec_url...";
-    $json = file_get_contents($spec_url);
+    echo "Fetching {$spec_url}...";
+    $spec_data = fetch_json($spec_url, $cache);
     echo "\n";
 
-    $data = json_decode($json, true);
-    if (json_last_error()) {
-        echo " JSON parse error: " . json_last_error_msg() . "\n";
+    echo "Fetching {$latest_release_url}...";
+    $latest_release = fetch_json($latest_release_url, $cache);
+    if (!isset($latest_release['name'])) {
+        echo " Missing release name\n";
         exit -1;
     }
+    echo "\n";
+
+    $latest_bundle_config_url = sprintf($bundle_config_url, $latest_release['name']);
+    echo "Fetching {$latest_bundle_config_url}...";
+    $bundle_config = fetch_json($latest_bundle_config_url, $cache);
+    echo "\n";
 
     echo 'Generating spec';
     $specGenerator->generate(
-        json_decode($json, true),
+        $spec_data,
+        $bundle_config,
         'AmpProject\Validator',
         $destination
     );

bin/src/Validator/SpecGenerator.php

@@ -6,6 +6,7 @@
 use AmpProject\Tooling\Validator\SpecGenerator\FileManager;
 use AmpProject\Tooling\Validator\SpecGenerator\Section;
 use AmpProject\Tooling\Validator\SpecGenerator\SpecPrinter;
+use Exception;
 use Nette\PhpGenerator\ClassType;
 use Nette\PhpGenerator\PhpNamespace;
 
@@ -24,11 +25,15 @@ final class SpecGenerator
      * Generate the PHP spec from a JSON definition.
      *
      * @param array  $jsonSpec      Validator spec as defined by the downloaded JSON file.
+     * @param array  $bundlesConfig Bundles configuration.
      * @param string $rootNamespace Root namespace to generate the PHP validator spec under.
      * @param string $destination   Destination folder to store the PHP validator spec under.
      */
-    public function generate($jsonSpec, $rootNamespace, $destination)
+    public function generate($jsonSpec, $bundlesConfig, $rootNamespace, $destination)
     {
+
+        $extensionsMeta = $this->gatherExtensionsMeta($bundlesConfig);
+
         $printer = new SpecPrinter();
         $printer->setTypeResolving(false);
 
@@ -55,6 +60,8 @@ public function generate($jsonSpec, $rootNamespace, $destination)
         $this->generateEntityClass('Error', $fileManager);
         $this->generateEntityClass('Tag', $fileManager);
         $this->generateEntityClass('TagWithExtensionSpec', $fileManager, 'interface');
+        $this->generateEntityClass('AggregateTag', $fileManager);
+        $this->generateEntityClass('AggregateTagWithExtensionSpec', $fileManager);
         $this->generateEntityClass('ExtensionSpec', $fileManager, 'trait');
         $this->generateEntityClass('Identifiable', $fileManager, 'interface');
         $this->generateEntityClass('IterableSection', $fileManager, 'interface');
@@ -86,7 +93,12 @@ public function generate($jsonSpec, $rootNamespace, $destination)
                           ->addComment("@return string");
                     break;
                 default:
-                    $sectionClassName = $this->generateSectionClass($section, $sectionSpec, $fileManager);
+                    $sectionClassName = $this->generateSectionClass(
+                        $section,
+                        $sectionSpec,
+                        $fileManager,
+                        $extensionsMeta
+                    );
 
                     $class->addProperty($section)
                           ->setPrivate()
@@ -151,9 +163,10 @@ private function generateEntityClass($entity, FileManager $fileManager, $type =
      * @param string      $section     Key of the section to generate.
      * @param mixed       $sectionSpec Spec data of the section to be generated.
      * @param FileManager $fileManager FileManager instance to use.
+     * @param array       $extensionsMeta Extensions meta.
      * @return string Section class name.
      */
-    private function generateSectionClass($section, $sectionSpec, FileManager $fileManager)
+    private function generateSectionClass($section, $sectionSpec, FileManager $fileManager, $extensionsMeta)
     {
         list($file, $namespace) = $fileManager->createNewNamespacedFile('Spec\\Section');
         $className = $this->getClassName($section);
@@ -163,8 +176,13 @@ private function generateSectionClass($section, $sectionSpec, FileManager $fileM
         $sectionProcessorClass = self::GENERATOR_NAMESPACE . "\\Section\\{$className}";
 
         if (class_exists($sectionProcessorClass)) {
-            /** @var Section $sectionProcessor */
-            $sectionProcessor = new $sectionProcessorClass();
+            if (Section\Tags::class === $sectionProcessorClass) {
+                $sectionProcessor = new Section\Tags($extensionsMeta);
+            } else {
+                /** @var Section $sectionProcessor */
+                $sectionProcessor = new $sectionProcessorClass();
+            }
+
             $sectionProcessor->process($fileManager, $sectionSpec, $namespace, $class);
         }
 
@@ -254,6 +272,63 @@ private function adaptJsonSpec($jsonSpec)
         return $jsonSpec;
     }
 
+    /**
+     * Gather extensions meta.
+     *
+     * @param array $bundlesConfig Bundles config.
+     * @return array Extensions meta.
+     */
+    private function gatherExtensionsMeta($bundlesConfig)
+    {
+        $extensions = [];
+        foreach ($bundlesConfig as $bundleConfig) {
+            if (! isset($bundleConfig['name']) || ! is_string($bundleConfig['name'])) {
+                throw new Exception('Missing name in bundles.config.extensions.json');
+            }
+            if (! isset($bundleConfig['latestVersion']) || ! is_string($bundleConfig['latestVersion'])) {
+                throw new Exception('Missing string latestVersion in bundles.config.extensions.json');
+            }
+            if (
+                ! isset($bundleConfig['version'])
+                ||
+                ! (is_string($bundleConfig['version']) || is_array($bundleConfig['version']) )
+            ) {
+                throw new Exception('Missing string/array version in bundles.config.extensions.json');
+            }
+
+            if (!isset($extensions[$bundleConfig['name']])) {
+                $extensions[$bundleConfig['name']] = [
+                    'versions' => [],
+                    'latestVersion' => null,
+                ];
+            }
+
+            $versions = (array) $bundleConfig['version'];
+            foreach ($versions as $version) {
+                if (array_key_exists($version, $extensions[$bundleConfig['name']]['versions'])) {
+                    throw new Exception("Version {$version} already seen for extension {$bundleConfig['name']}.");
+                }
+                $extensions[$bundleConfig['name']]['versions'][$version] = [
+                    'hasCss'   => ! empty($bundleConfig['options']['hasCss']),
+                    'hasBento' => (
+                        isset($bundleConfig['options']['wrapper']) && 'bento' === $bundleConfig['options']['wrapper']
+                    ),
+                ];
+
+                if (
+                    version_compare(
+                        $bundleConfig['latestVersion'],
+                        $extensions[$bundleConfig['name']]['latestVersion'],
+                        '>'
+                    )
+                ) {
+                    $extensions[$bundleConfig['name']]['latestVersion'] = $bundleConfig['latestVersion'];
+                }
+            }
+        }
+        return $extensions;
+    }
+
     /**
      * Collect all spec rule keys.
      *

bin/src/Validator/SpecGenerator/FileManager.php

@@ -174,8 +174,8 @@ private function addMissingImports(PhpFile $file)
                     $classes = array_merge($classes, $methodClasses);
                 }
 
-                $classes = array_merge($classes, $class->getExtends());
-                $classes = array_merge($classes, $class->getImplements());
+                $classes = array_merge($classes, (array) $class->getExtends());
+                $classes = array_merge($classes, (array) $class->getImplements());
             }
 
             foreach (array_unique($classes) as $class) {

bin/src/Validator/SpecGenerator/Section/Tags.php

@@ -17,6 +17,19 @@ final class Tags implements Section
     use ConstantNames;
     use MagicPropertyAnnotations;
 
+    /** @var array */
+    private $extensionsMeta;
+
+    /**
+     * Tags constructor.
+     *
+     * @param array $extensionsMeta Extensions meta.
+     */
+    public function __construct($extensionsMeta)
+    {
+        $this->extensionsMeta = $extensionsMeta;
+    }
+
     /**
      * Process a section.
      *
@@ -37,6 +50,8 @@ public function process(FileManager $fileManager, $spec, PhpNamespace $namespace
         $namespace->addUse("LogicException");
         $namespace->addUse("{$fileManager->getRootNamespace()}\\Spec\\IterableSection");
         $namespace->addUse("{$fileManager->getRootNamespace()}\\Spec\\Iteration");
+        $namespace->addUse("{$fileManager->getRootNamespace()}\\Spec\\AggregateTag");
+        $namespace->addUse("{$fileManager->getRootNamespace()}\\Spec\\AggregateTagWithExtensionSpec");
         $namespace->addUse("{$fileManager->getRootNamespace()}\\Spec\\Tag");
         $namespace->addUse("{$fileManager->getRootNamespace()}\\Spec\\TagWithExtensionSpec");
 
@@ -127,9 +142,18 @@ public function process(FileManager $fileManager, $spec, PhpNamespace $namespace
             }
 
             if (array_key_exists('extensionSpec', $tags[$tagId])) {
-                $extensionSpec                   = $tags[$tagId]['extensionSpec'];
-                $extensionName                   = $this->getKeyString($extensionSpec['name']);
-                $byExtensionSpec[$extensionName] = $tagIdString;
+                $extensionSpec = $tags[$tagId]['extensionSpec'];
+                $extensionSpec = $this->getKeyString($extensionSpec['name']);
+                if (array_key_exists($extensionSpec, $byExtensionSpec)) {
+                    if (!is_array($byExtensionSpec[$extensionSpec])) {
+                        $previousTagId                     = $byExtensionSpec[$extensionSpec];
+                        $byExtensionSpec[$extensionSpec]   = [];
+                        $byExtensionSpec[$extensionSpec][] = $previousTagId;
+                    }
+                    $byExtensionSpec[$extensionSpec][] = $tagIdString;
+                } else {
+                    $byExtensionSpec[$extensionSpec] = $tagIdString;
+                }
             }
         }
 
@@ -156,9 +180,9 @@ public function process(FileManager $fileManager, $spec, PhpNamespace $namespace
 
         $class->addConstant('BY_EXTENSION_SPEC', $byExtensionSpec)
             ->addComment(
-                "Mapping of extension name to tag ID.\n\n"
+                "Mapping of extension name to tag ID or array of tag IDs.\n\n"
                 . "This is used to optimize querying by extension spec.\n\n"
-                . "@var array<string>"
+                . "@var array<string|array<string>>"
             );
     }
 
@@ -274,6 +298,27 @@ private function generateTagSpecificClass($tagId, $jsonSpec, FileManager $fileMa
             $class->addConstant('EXTENSION_SPEC', $extensionSpec)
                   ->addComment("Array of extension spec rules.\n\n@var array");
 
+            $latestVersion = null;
+            if (isset($this->extensionsMeta[$extensionSpec['name']]['latestVersion'])) {
+                $latestVersion = $this->extensionsMeta[$extensionSpec['name']]['latestVersion'];
+            }
+
+            $class->addConstant('LATEST_VERSION', $latestVersion)
+                  ->addComment("Latest version of the extension.\n\n@var string");
+
+            if (isset($extensionSpec['version']) && isset($this->extensionsMeta[$extensionSpec['name']])) {
+                $versionsMeta = [];
+                foreach ($extensionSpec['version'] as $validVersion) {
+                    if (isset($this->extensionsMeta[$extensionSpec['name']]['versions'][$validVersion])) {
+                        $versionsMeta[$validVersion] =
+                            $this->extensionsMeta[$extensionSpec['name']]['versions'][$validVersion];
+                    }
+                }
+
+                $class->addConstant('VERSIONS_META', $versionsMeta)
+                      ->addComment("Meta data about the specific versions.\n\n@var array");
+            }
+
             $namespace->addUse("{$fileManager->getRootNamespace()}\\Spec\\TagWithExtensionSpec");
             $class->addImplement("{$fileManager->getRootNamespace()}\\Spec\\TagWithExtensionSpec");
             $namespace->addUse("{$fileManager->getRootNamespace()}\\Spec\\ExtensionSpec");