Change package path from {packagename}-{version} to {packagename}/{version}

.ci/jenkins-go-agent/Dockerfile

@@ -1,9 +1,9 @@
-FROM golang:1.12
+FROM golang:1.13
 
 RUN \
  apt-get update \
  && apt-get install -y --no-install-recommends \
- zip \
+ zip rsync\
  && rm -rf /var/lib/apt/lists/*
 
 RUN go get -u github.com/magefile/mage \

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Breaking changes
 
+* Change package path from /package/{packagename}-{version} to /package/{packagename}/{version} [#](https://github.com/elastic/integrations-registry/pull/)
 
 ### Bugfixes
 

Dockerfile

@@ -4,7 +4,7 @@ FROM golang:${GO_VERSION:-1.13.4}
 RUN \
  apt-get update \
  && apt-get install -y --no-install-recommends \
- zip \
+ zip rsync \
  && rm -rf /var/lib/apt/lists/*
 
 COPY ./ /go/src/github.com/elastic/package-registry

Jenkinsfile

@@ -3,7 +3,7 @@
 @Library('apm@current') _
 
 pipeline {
- agent { label 'docker && linux && immutable' }
+ agent { label 'ubuntu && immutable' }
  environment {
  BASE_DIR="src/github.com/elastic/package-registry"
  JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
@@ -27,8 +27,8 @@ pipeline {
  }
  stages {
  /**
- Checkout the code and stash it, to use it on other stages.
- */
+  Checkout the code and stash it, to use it on other stages.
+  */
  stage('Checkout') {
  steps {
  deleteDir()
@@ -37,8 +37,8 @@ pipeline {
  }
  }
  /**
- Checks formatting / linting.
- */
+  Checks formatting / linting.
+  */
  stage('Lint') {
  steps {
  deleteDir()
@@ -51,8 +51,8 @@ pipeline {
  }
  }
  /**
- Build the project from code..
- */
+  Build the project from code..
+  */
  stage('Build') {
  steps {
  deleteDir()
@@ -65,8 +65,8 @@ pipeline {
  }
  }
  /**
- Execute unit tests.
- */
+  Execute unit tests.
+  */
  stage('Test') {
  steps {
  deleteDir()
@@ -86,8 +86,8 @@ pipeline {
  }
  }
  /**
-  Publish Docker images.
- */
+ Publish Docker images.
+  */
  stage('Publish Docker image'){
  environment {
  DOCKER_IMG_TAG = "${env.DOCKER_IMG}:${env.GIT_BASE_COMMIT}"
@@ -105,8 +105,8 @@ pipeline {
  script: "docker push ${env.DOCKER_IMG_TAG}")
  sh(label: 'Re-tag Docker image',
  script: "docker tag ${env.DOCKER_IMG_TAG} ${env.DOCKER_IMG_TAG_BRANCH}")
-  sh(label: 'Push Docker image name',
-  script: "docker push ${env.DOCKER_IMG_TAG_BRANCH}")
+ sh(label: 'Push Docker image name',
+ script: "docker push ${env.DOCKER_IMG_TAG_BRANCH}")
  }
  }
  }

README.md

@@ -9,8 +9,8 @@ Endpoints:
 * `/`: Info about the registry
 * `/search`: Search for packages. By default returns all the most recent packages available.
 * `/categories`: List of the existing package categories and how many packages are in each category.
-* `/package/{name}-{version}`: Info about a package
-* `/package/{name}-{version}.tar.gz`: Download a package
+* `/package/{name}/{version}`: Info about a package
+* `/epr/{name}/{name}-{version}.tar.gz`: Download a package
 
 Examples for each API endpoint can be found here: https://github.com/elastic/package-registry/tree/master/docs/api
 

dev/generator/main.go

@@ -12,6 +12,7 @@ import (
  "log"
  "os"
  "path/filepath"
+ "strings"
 
  "github.com/magefile/mage/sh"
 
@@ -75,9 +76,9 @@ func Build(sourceDir, publicDir string) error {
 
 // CopyPackage copies the files of a package to the public directory
 func CopyPackage(src, dst string) error {
- fmt.Println(">> Copy package: " + src)
+ log.Println(">> Copy package: " + src)
  os.MkdirAll(dst, 0755)
- err := sh.RunV("cp", "-a", src, dst)
+ err := sh.RunV("rsync", "-a", src, dst)
  if err != nil {
  return err
  }
@@ -100,14 +101,19 @@ func BuildPackages(sourceDir, packagesPath string) error {
  continue
  }
 
+ // Finds the last occurence of "-" and then below splits it up in 2 parts.
+ dashIndex := strings.LastIndex(packageName, "-")
+ dstDir := filepath.Join(packagesPath, packageName[0:dashIndex], packageName[dashIndex+1:])
+
  if copy {
- err := CopyPackage(filepath.Join(sourceDir, packageName), packagesPath)
+ // Trailing slash is to make sure content of package is copied
+ err := CopyPackage(filepath.Join(sourceDir, packageName)+"/", dstDir)
  if err != nil {
  return err
  }
  }
 
- p, err := util.NewPackage(filepath.Join(packagesPath, packageName))
+ p, err := util.NewPackage(dstDir)
  if err != nil {
  return err
  }
@@ -200,15 +206,35 @@ func buildPackage(packagesBasePath string, p util.Package) error {
  }
 
  if tarGz {
- err = os.MkdirAll(filepath.Join(packagesBasePath, "..", "epr", p.Name), 0755)
+ tarGzDirPath := filepath.Join(packagesBasePath, "..", "epr", p.Name)
+ err = os.MkdirAll(tarGzDirPath, 0755)
+ if err != nil {
+ return err
+ }
+
+ tarGzName := p.Name + "-" + p.Version
+ copiedPackagePath := filepath.Join(tarGzDirPath, tarGzName)
+
+ // As the package directories are now {packagename}/{version} when just running tar, the dir inside
+ // the package had the wrong name. Using `-s` or `--transform` for some reason worked on the command line
+ // but not when run through Golang. So the hack for now is to just copy over all files with the correct name
+ // and then run tar on it.
+ // This could become even useful in the future as things like images or videos should potentially not be part of
+ // a tar.gz to keep it small.
+ err := CopyPackage(packagesBasePath+"/"+p.Name+"/"+p.Version+"/", copiedPackagePath)
  if err != nil {
  return err
  }
 
- err = sh.RunV("tar", "czf", filepath.Join(packagesBasePath, "..", "epr", p.Name, p.GetPath()+".tar.gz"), "-C", packagesBasePath+"/", filepath.Base(p.GetPath())+"/")
+ err = sh.RunV("tar", "czf", filepath.Join(packagesBasePath, "..", "epr", p.Name, tarGzName+".tar.gz"), "-C", tarGzDirPath, tarGzName+"/")
  if err != nil {
  return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err)
  }
+
+ err = os.RemoveAll(copiedPackagePath)
+ if err != nil {
+ return err
+ }
  }
 
  return nil

dev/import-beats/datasources.go

@@ -6,6 +6,7 @@ package main
 
 import (
  "fmt"
+
  "github.com/elastic/package-registry/util"
 )
 

docs/api/package.json

@@ -33,25 +33,25 @@
  }
  ],
  "assets": [
- "/package/example-1.0.0/manifest.yml",
- "/package/example-1.0.0/docs/README.md",
- "/package/example-1.0.0/img/icon.png",
- "/package/example-1.0.0/img/kibana-envoyproxy.jpg",
- "/package/example-1.0.0/dataset/foo/manifest.yml",
- "/package/example-1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
- "/package/example-1.0.0/kibana/index-pattern/filebeat-*.json",
- "/package/example-1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
- "/package/example-1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
- "/package/example-1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
- "/package/example-1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
- "/package/example-1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
- "/package/example-1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
- "/package/example-1.0.0/dataset/foo/fields/stream.yml",
- "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-entry.json",
- "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-http.json",
- "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-json.json",
- "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
- "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json"
+ "/package/example/1.0.0/manifest.yml",
+ "/package/example/1.0.0/docs/README.md",
+ "/package/example/1.0.0/img/icon.png",
+ "/package/example/1.0.0/img/kibana-envoyproxy.jpg",
+ "/package/example/1.0.0/dataset/foo/manifest.yml",
+ "/package/example/1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
+ "/package/example/1.0.0/kibana/index-pattern/filebeat-*.json",
+ "/package/example/1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
+ "/package/example/1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
+ "/package/example/1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
+ "/package/example/1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
+ "/package/example/1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
+ "/package/example/1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
+ "/package/example/1.0.0/dataset/foo/fields/stream.yml",
+ "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-entry.json",
+ "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-http.json",
+ "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-json.json",
+ "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
+ "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json"
  ],
  "datasets": [
  {
@@ -106,5 +106,5 @@
  }
  ],
  "download": "/epr/example/example-1.0.0.tar.gz",
- "path": "/package/example-1.0.0"
-}
+ "path": "/package/example/1.0.0"
+}

docs/api/search-category-logs.json

@@ -3,7 +3,7 @@
  "description": "Package with data sources",
  "download": "/epr/datasources/datasources-1.0.0.tar.gz",
  "name": "datasources",
- "path": "/package/datasources-1.0.0",
+ "path": "/package/datasources/1.0.0",
  "title": "Default datasource Integration",
  "type": "integration",
  "version": "1.0.0"
@@ -12,7 +12,7 @@
  "description": "Tests if no pipeline is set, it defaults to the default one",
  "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
  "name": "default-pipeline",
- "path": "/package/default-pipeline-0.0.2",
+ "path": "/package/default-pipeline/0.0.2",
  "title": "Default pipeline Integration",
  "type": "integration",
  "version": "0.0.2"
@@ -21,7 +21,7 @@
  "description": "This is the example integration",
  "download": "/epr/example/example-1.0.0.tar.gz",
  "name": "example",
- "path": "/package/example-1.0.0",
+ "path": "/package/example/1.0.0",
  "title": "Example Integration",
  "type": "integration",
  "version": "1.0.0"

docs/api/search-category-metrics.json

@@ -3,7 +3,7 @@
  "description": "This is the example integration",
  "download": "/epr/example/example-1.0.0.tar.gz",
  "name": "example",
- "path": "/package/example-1.0.0",
+ "path": "/package/example/1.0.0",
  "title": "Example Integration",
  "type": "integration",
  "version": "1.0.0"
@@ -12,7 +12,7 @@
  "description": "This is the foo integration",
  "download": "/epr/foo/foo-1.0.0.tar.gz",
  "name": "foo",
- "path": "/package/foo-1.0.0",
+ "path": "/package/foo/1.0.0",
  "title": "Foo",
  "type": "solution",
  "version": "1.0.0"

docs/api/search-kibana652.json

@@ -3,7 +3,7 @@
  "description": "Package with data sources",
  "download": "/epr/datasources/datasources-1.0.0.tar.gz",
  "name": "datasources",
- "path": "/package/datasources-1.0.0",
+ "path": "/package/datasources/1.0.0",
  "title": "Default datasource Integration",
  "type": "integration",
  "version": "1.0.0"
@@ -12,7 +12,7 @@
  "description": "Tests if no pipeline is set, it defaults to the default one",
  "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
  "name": "default-pipeline",
- "path": "/package/default-pipeline-0.0.2",
+ "path": "/package/default-pipeline/0.0.2",
  "title": "Default pipeline Integration",
  "type": "integration",
  "version": "0.0.2"
@@ -21,7 +21,7 @@
  "description": "This is the example integration.",
  "download": "/epr/example/example-0.0.2.tar.gz",
  "name": "example",
- "path": "/package/example-0.0.2",
+ "path": "/package/example/0.0.2",
  "title": "Example",
  "type": "integration",
  "version": "0.0.2"

docs/api/search-kibana721.json

@@ -3,7 +3,7 @@
  "description": "Package with data sources",
  "download": "/epr/datasources/datasources-1.0.0.tar.gz",
  "name": "datasources",
- "path": "/package/datasources-1.0.0",
+ "path": "/package/datasources/1.0.0",
  "title": "Default datasource Integration",
  "type": "integration",
  "version": "1.0.0"
@@ -12,7 +12,7 @@
  "description": "Tests if no pipeline is set, it defaults to the default one",
  "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
  "name": "default-pipeline",
- "path": "/package/default-pipeline-0.0.2",
+ "path": "/package/default-pipeline/0.0.2",
  "title": "Default pipeline Integration",
  "type": "integration",
  "version": "0.0.2"
@@ -21,7 +21,7 @@
  "description": "This is the example integration",
  "download": "/epr/example/example-1.0.0.tar.gz",
  "name": "example",
- "path": "/package/example-1.0.0",
+ "path": "/package/example/1.0.0",
  "title": "Example Integration",
  "type": "integration",
  "version": "1.0.0"
@@ -30,7 +30,7 @@
  "description": "This is the foo integration",
  "download": "/epr/foo/foo-1.0.0.tar.gz",
  "name": "foo",
- "path": "/package/foo-1.0.0",
+ "path": "/package/foo/1.0.0",
  "title": "Foo",
  "type": "solution",
  "version": "1.0.0"

docs/api/search-package-example.json

@@ -3,7 +3,7 @@
  "description": "This is the example integration.",
  "download": "/epr/example/example-0.0.2.tar.gz",
  "name": "example",
- "path": "/package/example-0.0.2",
+ "path": "/package/example/0.0.2",
  "title": "Example",
  "type": "integration",
  "version": "0.0.2"
@@ -12,7 +12,7 @@
  "description": "This is the example integration",
  "download": "/epr/example/example-1.0.0.tar.gz",
  "name": "example",
- "path": "/package/example-1.0.0",
+ "path": "/package/example/1.0.0",
  "title": "Example Integration",
  "type": "integration",
  "version": "1.0.0"