Merge pull request #89 from nspcc-dev/723-rest-gw

env: add rest-gateway

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "neofs-testlib"
-version = "1.1.19"
+version = "1.1.20"
 description = "Building blocks and utilities to facilitate development of automated tests for NeoFS system"
 readme = "README.md"
 authors = [{ name = "NSPCC", email = "info@nspcc.ru" }]
@@ -50,7 +50,7 @@ line-length = 100
 target-version = ["py310"]
 
 [tool.bumpver]
-current_version = "1.1.19"
+current_version = "1.1.20"
 version_pattern = "MAJOR.MINOR.PATCH"
 commit_message  = "Bump version {old_version} -> {new_version}"
 commit          = true

pytest_tests/test_env.py

@@ -140,8 +140,8 @@ def test_s3_gw_put_get(neofs_env: NeoFSEnv, s3_creds, wallet: NodeWallet):
     s3_client.put_object(**{"Body": file_content, "Bucket": bucket_name, "Key": filekey})
     s3_client.get_object(**{"Bucket": bucket_name, "Key": filekey})
 
-
-def test_http_gw_put_get(neofs_env: NeoFSEnv, wallet: NodeWallet, zero_fee):
+@pytest.mark.parametrize("gw_type", ["HTTP", "REST"])
+def test_gateways_put_get(neofs_env: NeoFSEnv, wallet: NodeWallet, zero_fee, gw_type):
     cli = neofs_env.neofs_cli(neofs_env.generate_cli_config(wallet))
 
     result = cli.container.create(
@@ -166,14 +166,17 @@ def test_http_gw_put_get(neofs_env: NeoFSEnv, wallet: NodeWallet, zero_fee):
     with open(filename, "w") as file:
         file.write("123456789")
 
-    request = f"http://{neofs_env.http_gw.address}/upload/{cid}"
+    if gw_type == "HTTP":
+        request = f"http://{neofs_env.http_gw.address}/upload/{cid}"
+    else:
+        request = f"http://{neofs_env.rest_gw.address}/v1/upload/{cid}"
     files = {"upload_file": open(filename, "rb")}
     body = {"filename": filename}
     resp = requests.post(request, files=files, data=body)
 
     if not resp.ok:
         raise Exception(
-            f"""Failed to get object via HTTP gate:
+            f"""Failed to get object via {gw_type} gate:
                 request: {resp.request.path_url},
                 response: {resp.text},
                 status code: {resp.status_code} {resp.reason}"""
@@ -182,13 +185,16 @@ def test_http_gw_put_get(neofs_env: NeoFSEnv, wallet: NodeWallet, zero_fee):
     oid = resp.json().get("object_id")
 
     download_attribute = "?download=true"
-    request = f"http://{neofs_env.http_gw.address}/get/{cid}/{oid}{download_attribute}"
+    if gw_type == "HTTP":
+        request = f"http://{neofs_env.http_gw.address}/get/{cid}/{oid}{download_attribute}"
+    else:
+        request = f"http://{neofs_env.rest_gw.address}/v1/get/{cid}/{oid}{download_attribute}"
 
     resp = requests.get(request, stream=True)
 
     if not resp.ok:
         raise Exception(
-            f"""Failed to get object via HTTP gate:
+            f"""Failed to get object via {gw_type} gate:
                 request: {resp.request.path_url},
                 response: {resp.text},
                 status code: {resp.status_code} {resp.reason}"""

src/neofs_testlib/__init__.py

@@ -1 +1 @@
-__version__ = "1.1.19"
+__version__ = "1.1.20"

src/neofs_testlib/env/env.py

@@ -138,6 +138,11 @@ def deploy_http_gw(self):
         self.http_gw = HTTP_GW(self)
         self.http_gw.start()
 
+    @allure.step("Deploy rest gateway")
+    def deploy_rest_gw(self):
+        self.rest_gw = REST_GW(self)
+        self.rest_gw.start()
+
     @allure.step("Generate wallet")
     def generate_wallet(
         self,
@@ -179,6 +184,7 @@ def generate_wallet(
 
     @allure.step("Kill current neofs env")
     def kill(self):
+        self.rest_gw.process.kill()
         self.http_gw.process.kill()
         self.s3_gw.process.kill()
         for sn in self.storage_nodes:
@@ -213,6 +219,7 @@ def simple(cls) -> "NeoFSEnv":
         )
         neofs_env.deploy_s3_gw()
         neofs_env.deploy_http_gw()
+        neofs_env.deploy_rest_gw()
         return neofs_env
 
     @staticmethod
@@ -612,6 +619,7 @@ def start(self):
         self._generate_config()
         logger.info(f"Launching HTTP GW: {self}")
         self._launch_process()
+        logger.info(f"Launched HTTP GW: {self}")
 
     def _generate_config(self):
         NeoFSEnv.generate_config_file(
@@ -638,3 +646,74 @@ def _launch_process(self):
             stderr=stderr_fp,
             env=http_gw_env,
         )
+
+
+class REST_GW:
+    def __init__(self, neofs_env: NeoFSEnv):
+        self.neofs_env = neofs_env
+        self.config_path = NeoFSEnv._generate_temp_file(extension="yml")
+        self.wallet = NodeWallet(
+            path=NeoFSEnv._generate_temp_file(),
+            address="",
+            password=self.neofs_env.default_password,
+        )
+        self.address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.pprof_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.metrics_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.stdout = "Not initialized"
+        self.stderr = "Not initialized"
+        self.process = None
+
+    def __str__(self):
+        return f"""
+            REST Gateway:
+            - Address: {self.address}
+            - Pprof address: {self.pprof_address}
+            - Metrics address: {self.metrics_address}
+            - REST GW Config path: {self.config_path}
+            - STDOUT: {self.stdout}
+            - STDERR: {self.stderr}
+        """
+
+    def __getstate__(self):
+        attributes = self.__dict__.copy()
+        del attributes["process"]
+        return attributes
+
+    def start(self):
+        if self.process is not None:
+            raise RuntimeError(f"This rest gw instance has already been started:\n{self}")
+        self.neofs_env.generate_wallet(WalletType.STORAGE, self.wallet, label=f"rest")
+        logger.info(f"Generating config for rest gw at {self.config_path}")
+        self._generate_config()
+        logger.info(f"Launching REST GW: {self}")
+        self._launch_process()
+        logger.info(f"Launched REST GW: {self}")
+
+    def _generate_config(self):
+        NeoFSEnv.generate_config_file(
+            config_template="rest.yaml",
+            config_path=self.config_path,
+            address=self.address,
+            wallet=self.wallet,
+            pprof_address=self.pprof_address,
+            metrics_address=self.metrics_address,
+        )
+
+    def _launch_process(self):
+        self.stdout = NeoFSEnv._generate_temp_file()
+        self.stderr = NeoFSEnv._generate_temp_file()
+        stdout_fp = open(self.stdout, "w")
+        stderr_fp = open(self.stderr, "w")
+        rest_gw_env = {}
+
+        for index, sn in enumerate(self.neofs_env.storage_nodes):
+            rest_gw_env[f"REST_GW_POOL_PEERS_{index}_ADDRESS"] = sn.endpoint
+            rest_gw_env[f"REST_GW_POOL_PEERS_{index}_WEIGHT"] = "0.2"
+
+        self.process = subprocess.Popen(
+            [self.neofs_env.neofs_rest_gw_path, "--config", self.config_path],
+            stdout=stdout_fp,
+            stderr=stderr_fp,
+            env=rest_gw_env,
+        )

src/neofs_testlib/env/templates/network.yaml

@@ -27,3 +27,4 @@ storage:
   sn4: {{ default_password }}
   s3: {{ default_password }}
   http: {{ default_password }}
+  rest: {{ default_password }}

src/neofs_testlib/env/templates/rest.yaml

@@ -0,0 +1,47 @@
+logger:
+  level: debug # Log level
+
+# Wallet settings
+wallet:
+  path: {{ wallet.path }}  # Path to wallet 
+  passphrase: {{ wallet.password }}  # Passphrase to decrypt wallet
+
+pprof:
+  enabled: true # Enable pprof.
+  address: {{ pprof_address }}
+prometheus:
+  enabled: true # Enable metrics.
+  address: {{ metrics_address }}
+
+pool:
+  # Timeout to dial node.
+  node-dial-timeout: 5s
+  # Timeout to check node health during rebalance.
+  healthcheck-timeout: 5s
+  # Interval to check nodes' health.
+  rebalance-timer: 30s
+  # The number of errors on connection after which node is considered as unhealthy.
+  error-threshold: 100
+
+server:
+  # The listeners to enable, this can be repeated and defaults to the schemes in the swagger spec.
+  scheme: [ http ]
+  # Grace period for which to wait before killing idle connections
+  cleanup-timeout: 10s
+  # Grace period for which to wait before shutting down the server
+  graceful-timeout: 15s
+  # Controls the maximum number of bytes the server will read parsing the request header's keys and values,
+  # including the request line. It does not limit the size of the request body.
+  max-header-size: 1000000
+
+  # The IP and port to listen on.
+  listen-address: {{ address }}
+  # Limit the number of outstanding requests.
+  listen-limit: 0
+  # Sets the TCP keep-alive timeouts on accepted connections.
+  # It prunes dead TCP connections ( e.g. closing laptop mid-download).
+  keep-alive: 3m
+  # Maximum duration before timing out read of the request.
+  read-timeout: 30s
+  # Maximum duration before timing out write of the response.
+  write-timeout: 30s