Skip to content

Commit

Permalink
repo: restructure the repository layout
Browse files Browse the repository at this point in the history
Restructure the whole repository layout to a more streamlined layout. This
commit focuses on the go code. Other code will be considered eventually.

The overall structure was discussed in #4076.

Each service and cli tools gets a top-level directory. Packages that are shared
across multiple applications are grouped in the `private` directory. This should
indicate that these packages are not intended to be used by external parties,
and that semantic versioning will not apply to these packages. Developer tools
are grouped in the `tools` directory.

Code that is intended for use by third external parties is grouped
in the `pkg` directory. Here we should strive for a stable package
API that does not change too often.

To smoothen transition, the following gist has all the metadata
of the move: https://gist.github.com/oncilla/96bdabb00359fdb4436a7a50d57d3cf3
You can use [go-imports.sh](https://gist.github.com/oncilla/96bdabb00359fdb4436a7a50d57d3cf3#file-go-imports-sh)
to fix the imports. [shuffle.yml](https://gist.github.com/oncilla/96bdabb00359fdb4436a7a50d57d3cf3#file-shuffle-yml)
lists all the moved packages and their targets.

GitOrigin-RevId: 1a49e3871a65a4bbeac515ada78f7208a85d7558
  • Loading branch information
oncilla committed Mar 12, 2022
1 parent e9f9b49 commit 15d455f
Show file tree
Hide file tree
Showing 2,431 changed files with 92,234 additions and 92,226 deletions.
2 changes: 1 addition & 1 deletion .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ test:unit --test_tag_filters=unit --build_tests_only --print_relative_test_log_p
test:unit_all --test_tag_filters=unit --build_tests_only --print_relative_test_log_paths //...

test:integration --test_tag_filters=integration --build_tests_only --print_relative_test_log_paths
test:integration_go --test_tag_filters=integration --build_tests_only --print_relative_test_log_paths //go/...
test:integration_go --test_tag_filters=integration --build_tests_only --print_relative_test_log_paths -- //... -//acceptance/... -//demo/...

test:lint --test_tag_filters=lint --build_tests_only --print_relative_test_log_paths --test_summary terse --test_output errors --noshow_progress --experimental_convenience_symlinks=ignore //...
4 changes: 2 additions & 2 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ steps:
- diff -u /tmp/test-artifacts/go.mod go.mod
- diff -u /tmp/test-artifacts/go.sum go.sum
- diff -u /tmp/test-artifacts/go_deps.bzl go_deps.bzl
- cp -R go/pkg/proto/ /tmp/test-artifacts
- cp -R pkg/proto/ /tmp/test-artifacts
- make protobuf
- diff -ur /tmp/test-artifacts/proto/ go/pkg/proto/
- diff -ur /tmp/test-artifacts/proto/ pkg/proto/
- mkdir -p /tmp/test-artifacts/licenses
- ./tools/licenses.sh /tmp/test-artifacts/licenses
- diff -rNu3 /tmp/test-artifacts/licenses ./licenses/data
Expand Down
36 changes: 18 additions & 18 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@ flake8_lint_config(
pkg_tar(
name = "scion",
srcs = [
"//go/cs",
"//go/daemon",
"//go/dispatcher",
"//go/posix-gateway",
"//go/posix-router",
"//go/scion",
"//go/scion-pki",
"//go/tools/pathdb_dump",
"//control/cmd/control",
"//daemon/cmd/daemon",
"//dispatcher/cmd/dispatcher",
"//gateway/cmd/gateway",
"//router/cmd/router",
"//scion-pki/cmd/scion-pki",
"//scion/cmd/scion",
"//tools/pathdb_dump",
],
mode = "0755",
package_dir = "",
Expand All @@ -59,15 +59,15 @@ pkg_tar(
pkg_tar(
name = "scion-ci",
srcs = [
"//go/acceptance/sig_ping_acceptance",
"//go/integration/braccept",
"//go/integration/end2end",
"//go/integration/end2end_integration",
"//go/integration/scion_integration",
"//go/lib/xtest/graphupdater",
"//go/pktgen",
"//go/tools/buildkite_artifacts",
"//go/tools/udpproxy",
"//acceptance/cmd/sig_ping_acceptance",
"//pkg/private/xtest/graphupdater",
"//tools/braccept",
"//tools/buildkite/cmd/buildkite_artifacts",
"//tools/end2end",
"//tools/end2end_integration",
"//tools/pktgen/cmd/pktgen",
"//tools/scion_integration",
"//tools/udpproxy",
],
mode = "0755",
package_dir = "",
Expand All @@ -77,7 +77,7 @@ pkg_tar(
pkg_tar(
name = "scion-topo",
srcs = [
"//go/scion-pki",
"//scion-pki/cmd/scion-pki",
],
mode = "0755",
package_dir = "",
Expand Down
14 changes: 7 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,12 @@ scion-topo:
tar --overwrite -xf bazel-bin/scion-topo.tar -C bin

protobuf:
rm -rf bazel-bin/go/pkg/proto/*/go_default_library_/github.com/scionproto/scion/go/pkg/proto/*
bazel build --output_groups=go_generated_srcs //go/pkg/proto/...
rm -f go/pkg/proto/*/*.pb.go
cp -r bazel-bin/go/pkg/proto/*/go_default_library_/github.com/scionproto/scion/go/pkg/proto/* go/pkg/proto
cp -r bazel-bin/go/pkg/proto/*/*/go_default_library_/github.com/scionproto/scion/go/pkg/proto/* go/pkg/proto
chmod 0644 go/pkg/proto/*/*.pb.go
rm -rf bazel-bin/pkg/proto/*/go_default_library_/github.com/scionproto/scion/pkg/proto/*
bazel build --output_groups=go_generated_srcs //pkg/proto/...
rm -f pkg/proto/*/*.pb.go
cp -r bazel-bin/pkg/proto/*/go_default_library_/github.com/scionproto/scion/pkg/proto/* pkg/proto
cp -r bazel-bin/pkg/proto/*/*/go_default_library_/github.com/scionproto/scion/pkg/proto/* pkg/proto
chmod 0644 pkg/proto/*/*.pb.go

mocks:
tools/gomocks
Expand All @@ -63,4 +63,4 @@ lint:
tools/lint.sh

golangci-lint:
docker run --rm -v "${PWD}:/src" -w /src golangci/golangci-lint:v1.43.0 golangci-lint run --config=/src/.golangcilint.yml --timeout=3m go/...
docker run --rm -v "${PWD}:/src" -w /src golangci/golangci-lint:v1.43.0 golangci-lint run --config=/src/.golangcilint.yml --timeout=3m --skip-dirs doc ./...
14 changes: 14 additions & 0 deletions acceptance/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
load("//lint:go.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = ["sigcmn.go"],
importpath = "github.com/scionproto/scion/acceptance",
visibility = ["//visibility:public"],
deps = [
"//pkg/addr:go_default_library",
"//pkg/private/serrors:go_default_library",
"//pkg/snet:go_default_library",
"//tools/integration:go_default_library",
],
)
4 changes: 2 additions & 2 deletions acceptance/cert_renewal/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ topogen_test(
src = "test.py",
args = [
"--end2end_integration",
"$(location //go/integration/end2end_integration)",
"$(location //tools/end2end_integration)",
],
data = ["//go/integration/end2end_integration"],
data = ["//tools/end2end_integration"],
topo = "//topology:tiny4.topo",
)
20 changes: 20 additions & 0 deletions acceptance/cmd/sig_ping_acceptance/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
load("//lint:go.bzl", "go_library")
load("//:scion.bzl", "scion_go_binary")

go_library(
name = "go_default_library",
srcs = ["main.go"],
importpath = "github.com/scionproto/scion/acceptance/cmd/sig_ping_acceptance",
visibility = ["//visibility:private"],
deps = [
"//acceptance:go_default_library",
"//pkg/log:go_default_library",
"//tools/integration:go_default_library",
],
)

scion_go_binary(
name = "sig_ping_acceptance",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)
87 changes: 87 additions & 0 deletions acceptance/cmd/sig_ping_acceptance/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
// Copyright 2018 ETH Zurich
// Copyright 2020 ETH Zurich, Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// The sig_ping acceptance test checks for basic connectivity between AS through the SIG, using
// standard ping.

package main

import (
"flag"
"fmt"
"os"
"strconv"
"time"

"github.com/scionproto/scion/acceptance"
"github.com/scionproto/scion/pkg/log"
"github.com/scionproto/scion/tools/integration"
)

var (
name = "sig_ping_acceptance"
cmd = "ping"
attempts = flag.Int("attempts", 5, "Number of ping attempts.")
fail = flag.Bool("fail", false, "Succeed if the pings don't make it through.")
timeoutFlag = flag.Int("timeout", 0, "The timeout that is passed to the ping command")
)

func main() {
os.Exit(realMain())
}

func realMain() int {
if err := integration.Init(); err != nil {
fmt.Fprintf(os.Stderr, "Failed to init: %s\n", err)
return 1
}
defer log.HandlePanic()
defer log.Flush()
if !*integration.Docker {
log.Error(fmt.Sprintf("Can only run %s test with docker!", name))
return 1
}
if err := acceptance.ReadTestingConf(); err != nil {
log.Error("Testing conf reading failed", "err", err)
return 1
}

args := []string{cmd, "-c", strconv.Itoa(*attempts), "-O", integration.DstHostReplace}
timeout := time.Duration(*attempts)*2*time.Second + integration.DefaultRunTimeout
if *timeoutFlag != 0 {
args = append(args, "-w", strconv.Itoa(*timeoutFlag))
timeout = time.Duration(*timeoutFlag)*time.Second + integration.DefaultRunTimeout
}
in := integration.NewBinaryIntegration(name, integration.WrapperCmd, args, nil)
err := integration.RunUnaryTests(
in,
integration.UniqueIAPairs(acceptance.SigAddr),
timeout,
nil,
)
if !*fail && err != nil {
// The pings were supposed to get through but they didn't.
fmt.Fprintf(os.Stderr, "Failed to run tests: %s\n", err)
return 1

}
if *fail && err == nil {
fmt.Fprintf(os.Stderr, "Failed to run tests: "+
"Pings were supposed to not to reach the destination but they did.\n")
return 1

}
return 0
}
4 changes: 2 additions & 2 deletions acceptance/common/topogen.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,13 @@ def topogen_test(
setup_params += " --sig"

common_args = [
"--executables=scion-pki:$(location //go/scion-pki)",
"--executables=scion-pki:$(location //scion-pki/cmd/scion-pki)",
"--executables=topogen:$(location //python/topology:topogen)",
"--topo=$(location %s)" % topo,
"--setup-params='%s'" % setup_params,
]
common_data = [
"//go/scion-pki",
"//scion-pki/cmd/scion-pki",
"//python/topology:topogen",
"//tools:docker_ip",
topo,
Expand Down
5 changes: 2 additions & 3 deletions acceptance/router_multi/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
load("@io_bazel_rules_docker//container:container.bzl", "container_image")
load("//lint:py.bzl", "py_library")
load("//acceptance/common:raw.bzl", "raw_test")

exports_files([
Expand All @@ -10,7 +9,7 @@ exports_files([

args = [
"--braccept_bin",
"$(location //go/integration/braccept)",
"$(location //tools/braccept)",
"--pause_tar",
"$(location //acceptance/router_multi:pause.tar)",
"--image_tar",
Expand All @@ -23,7 +22,7 @@ data = [
"pause.tar",
":conf",
":router.tar",
"//go/integration/braccept",
"//tools/braccept",
]

raw_test(
Expand Down
4 changes: 2 additions & 2 deletions acceptance/sig_ping/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ topogen_test(
src = "test.py",
args = [
"--gateway_acceptance",
"$(location //go/acceptance/sig_ping_acceptance)",
"$(location //acceptance/cmd/sig_ping_acceptance)",
],
data = [
"//go/acceptance/sig_ping_acceptance",
"//acceptance/cmd/sig_ping_acceptance",
],
gateway = True,
topo = "//topology:tiny4.topo",
Expand Down
6 changes: 3 additions & 3 deletions acceptance/sig_short_exp_time/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sh_test(

container_image(
name = "udpproxy",
base = "//go/tools/udpproxy",
base = "//tools/udpproxy",
)

container_image(
Expand Down Expand Up @@ -48,7 +48,7 @@ container_image(
"--config",
"/sig.toml",
],
entrypoint = ["/app/posix-gateway"],
entrypoint = ["/app/gateway"],
files = [
"testdata/1-ff00_0_110/sig/cfg.json",
"testdata/1-ff00_0_110/sig/fake_sciond.json",
Expand All @@ -63,7 +63,7 @@ container_image(
"--config",
"/sig.toml",
],
entrypoint = ["/app/posix-gateway"],
entrypoint = ["/app/gateway"],
files = [
"testdata/1-ff00_0_111/sig/cfg.json",
"testdata/1-ff00_0_111/sig/fake_sciond.json",
Expand Down
8 changes: 4 additions & 4 deletions go/acceptance/sigcmn.go → acceptance/sigcmn.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ import (
"os"
"strings"

"github.com/scionproto/scion/go/lib/addr"
"github.com/scionproto/scion/go/lib/integration"
"github.com/scionproto/scion/go/lib/serrors"
"github.com/scionproto/scion/go/lib/snet"
"github.com/scionproto/scion/pkg/addr"
"github.com/scionproto/scion/pkg/private/serrors"
"github.com/scionproto/scion/pkg/snet"
"github.com/scionproto/scion/tools/integration"
)

var (
Expand Down
10 changes: 5 additions & 5 deletions acceptance/topo_cs_reload/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ go_test(
srcs = ["reload_test.go"],
args = [
"-scion_pki",
"$(location //go/scion-pki:scion-pki)",
"$(location //scion-pki/cmd/scion-pki:scion-pki)",
"-topo",
"$(location testdata/test.topo)",
],
Expand All @@ -21,12 +21,12 @@ go_test(
":testdata/test.topo",
"//acceptance/topo_common:invalid_reloads",
"//acceptance/topo_common:topology",
"//go/scion-pki",
"//scion-pki/cmd/scion-pki",
],
tags = ["integration"],
deps = [
"//go/lib/topology:go_default_library",
"//go/lib/xtest:go_default_library",
"//pkg/private/xtest:go_default_library",
"//private/topology:go_default_library",
"@com_github_stretchr_testify//assert:go_default_library",
"@com_github_stretchr_testify//require:go_default_library",
"@io_bazel_rules_go//go/tools/bazel:go_default_library",
Expand All @@ -51,7 +51,7 @@ container_image(
"--config",
"/cs.toml",
],
entrypoint = ["/app/cs"],
entrypoint = ["/app/control"],
files = [
"testdata/cs.toml",
"testdata/tls.key",
Expand Down
8 changes: 4 additions & 4 deletions acceptance/topo_cs_reload/reload_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"io"
"net/http"
"os"
"os/exec"
Expand All @@ -29,8 +29,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/scionproto/scion/go/lib/topology"
"github.com/scionproto/scion/go/lib/xtest"
"github.com/scionproto/scion/pkg/private/xtest"
"github.com/scionproto/scion/private/topology"
)

var (
Expand Down Expand Up @@ -179,7 +179,7 @@ func fetchTopologyFromEndpoint(t *testing.T, url string) *topology.RWTopology {
resp, err := http.Get(url)
require.NoError(t, err)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var topo *topology.RWTopology
require.NoError(t, json.Unmarshal(body, &topo))
Expand Down
Loading

0 comments on commit 15d455f

Please sign in to comment.