Improve the developer experience when running tests (#163)

Migrate the PRs from the pulsar-client-cpp repo:
- apache/pulsar-client-cpp#297
- apache/pulsar-client-cpp#236
- apache/pulsar-client-cpp#340

Author: BewareMyPower

build-support/pulsar-test-service-start.sh

@@ -25,7 +25,7 @@ cd $SRC_DIR
 
 ./build-support/pulsar-test-service-stop.sh
 
-CONTAINER_ID=$(docker run -i -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600)
+CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600)
 echo $CONTAINER_ID > .tests-container-id.txt
 
 docker cp tests/test-conf $CONTAINER_ID:/pulsar/test-conf

build-support/start-test-service-inside-container.sh

@@ -34,8 +34,23 @@ bin/pulsar tokens create \
             --secret-key file:///pulsar/data/tokens/secret.key \
             > /pulsar/data/tokens/token.txt
 
+# Unset the HTTP proxy to avoid the REST requests being affected
+export http_proxy=
+TOKEN=$(bin/pulsar tokens create \
+            --subject superUser \
+            --secret-key file:///pulsar/data/tokens/secret.key)
+
+put() {
+    curl -H "Authorization: Bearer $TOKEN" \
+        -L http://localhost:8080/admin/v2/$1 \
+        -H 'Content-Type: application/json' \
+        -X PUT \
+        -d $(echo $2 | sed 's/ //g')
+}
+
 export PULSAR_STANDALONE_CONF=test-conf/standalone-ssl.conf
 export PULSAR_PID_DIR=/tmp
+sed -i 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml
 bin/pulsar-daemon start standalone \
         --no-functions-worker --no-stream-storage \
         --bookkeeper-dir data/bookkeeper
@@ -48,57 +63,53 @@ echo "-- Pulsar service is ready -- Configure permissions"
 export PULSAR_CLIENT_CONF=test-conf/client-ssl.conf
 
 # Create "standalone" cluster if it does not exist
-bin/pulsar-admin clusters list | grep -q '^standalone$' ||
-        bin/pulsar-admin clusters create \
-                standalone \
-                --url http://localhost:8080/ \
-                --url-secure https://localhost:8443/ \
-                --broker-url pulsar://localhost:6650/ \
-                --broker-url-secure pulsar+ssl://localhost:6651/
+put clusters/standalone '{
+  "serviceUrl": "http://localhost:8080/",
+  "serviceUrlTls": "https://localhost:8443/",
+  "brokerServiceUrl": "pulsar://localhost:6650/",
+  "brokerServiceUrlTls": "pulsar+ssl://localhost:6651/"
+}'
 
 # Update "public" tenant
-bin/pulsar-admin tenants create public -r "anonymous" -c "standalone"
+put tenants/public '{
+  "adminRoles": ["anonymous"],
+  "allowedClusters": ["standalone"]
+}'
 
 # Update "public/default" with no auth required
-bin/pulsar-admin namespaces create public/default -c standalone
-bin/pulsar-admin namespaces grant-permission public/default \
-                        --actions produce,consume \
-                        --role "anonymous"
+put namespaces/public/default '{
+  "auth_policies": {
+    "namespace_auth": {
+      "anonymous": ["produce", "consume"]
+    }
+  },
+  "replication_clusters": ["standalone"]
+}'
 
 # Create "public/default-2" with no auth required
-bin/pulsar-admin namespaces create public/default-2 \
-                        --clusters standalone
-bin/pulsar-admin namespaces grant-permission public/default-2 \
-                        --actions produce,consume \
-                        --role "anonymous"
-
-# Create "public/default-3" with no auth required
-bin/pulsar-admin namespaces create public/default-3 \
-                        --clusters standalone
-bin/pulsar-admin namespaces grant-permission public/default-3 \
-                        --actions produce,consume \
-                        --role "anonymous"
-
-# Create "public/default-4" with encryption required
-bin/pulsar-admin namespaces create public/default-4 \
-                        --clusters standalone
-bin/pulsar-admin namespaces grant-permission public/default-4 \
-                        --actions produce,consume \
-                        --role "anonymous"
-bin/pulsar-admin namespaces set-encryption-required public/default-4 -e
-
-# Create "public/test-backlog-quotas" to test backlog quotas policy
-bin/pulsar-admin namespaces create public/test-backlog-quotas \
-                        --clusters standalone
+put namespaces/public/default-2 '{
+  "auth_policies": {
+    "namespace_auth": {
+      "anonymous": ["produce", "consume"]
+    }
+  },
+  "replication_clusters": ["standalone"]
+}'
 
 # Create "private" tenant
-bin/pulsar-admin tenants create private -r "" -c "standalone"
+put tenants/private '{
+  "adminRoles": [],
+  "allowedClusters": ["standalone"]
+}'
 
 # Create "private/auth" with required authentication
-bin/pulsar-admin namespaces create private/auth --clusters standalone
-
-bin/pulsar-admin namespaces grant-permission private/auth \
-                        --actions produce,consume \
-                        --role "token-principal"
+put namespaces/private/auth '{
+  "auth_policies": {
+    "namespace_auth": {
+      "token-principal": ["produce", "consume"]
+    }
+  },
+  "replication_clusters": ["standalone"]
+}'
 
 echo "-- Ready to start tests"

tests/pulsar_test.py

@@ -56,7 +56,9 @@
 from urllib.request import urlopen, Request
 
 TM = 10000  # Do not wait forever in tests
-CERTS_DIR = os.path.dirname(os.path.abspath(__file__)) + "/test-conf/"
+TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+TOKEN_PATH = TESTS_DIR + "/.test-token.txt"
+CERTS_DIR = TESTS_DIR + "/test-conf/"
 
 def doHttpPost(url, data):
     req = Request(url, data.encode())
@@ -1247,7 +1249,7 @@ def test_get_topics_partitions(self):
         client.close()
 
     def test_token_auth(self):
-        with open(".test-token.txt") as tf:
+        with open(TOKEN_PATH) as tf:
             token = tf.read().strip()
 
         # Use adminUrl to test both HTTP request and binary protocol
@@ -1266,7 +1268,7 @@ def test_token_auth(self):
 
     def test_token_auth_supplier(self):
         def read_token():
-            with open(".test-token.txt") as tf:
+            with open(TOKEN_PATH) as tf:
                 return tf.read().strip()
 
         client = Client(self.serviceUrl, authentication=AuthenticationToken(read_token))