Skip to content

Commit

Permalink
feat: avoid creating unused ThreadPools (#91)
Browse files Browse the repository at this point in the history
* feat: avoid creating unused ThreadPools

* docs: add requested TODOs
  • Loading branch information
tomplus authored and k8s-ci-robot committed Dec 17, 2018
1 parent 9cf6542 commit 9d14077
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 0 deletions.
42 changes: 42 additions & 0 deletions openapi/python-api_client.py.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
--- a/kubernetes/client/api_client.py
+++ b/kubernetes/client/api_client.py
@@ -58,13 +58,15 @@ class ApiClient(object):
'datetime': datetime,
'object': object,
}
+ _pool = None

- def __init__(self, configuration=None, header_name=None, header_value=None, cookie=None):
+ def __init__(self, configuration=None, header_name=None, header_value=None,
+ cookie=None, pool_threads=None):
if configuration is None:
configuration = Configuration()
self.configuration = configuration
+ self.pool_threads = pool_threads

- self.pool = ThreadPool()
self.rest_client = RESTClientObject(configuration)
self.default_headers = {}
if header_name is not None:
@@ -74,8 +76,19 @@ class ApiClient(object):
self.user_agent = 'Swagger-Codegen/8.0.0-snapshot/python'

def __del__(self):
- self.pool.close()
- self.pool.join()
+ if self._pool:
+ self._pool.close()
+ self._pool.join()
+ self._pool = None
+
+ @property
+ def pool(self):
+ """Create thread pool on first request
+ avoids instantiating unused threadpool for blocking clients.
+ """
+ if self._pool is None:
+ self._pool = ThreadPool(self.pool_threads)
+ return self._pool

@property
def user_agent(self):
27 changes: 27 additions & 0 deletions openapi/python-asyncio-api_client.py.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
61a62
> _pool = None
64c65
< cookie=None):
---
> cookie=None, pool_threads=None):
67a69
> self.pool_threads = pool_threads
69d70
< self.pool = ThreadPool()
79,80c80,92
< self.pool.close()
< self.pool.join()
---
> if self._pool:
> self._pool.close()
> self._pool.join()
> self._pool = None
>
> @property
> def pool(self):
> """Create thread pool on first request
> avoids instantiating unused threadpool for blocking clients.
> """
> if self._pool is None:
> self._pool = ThreadPool(self.pool_threads)
> return self._pool
8 changes: 8 additions & 0 deletions openapi/python-asyncio.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,22 +66,30 @@ if [ ${PACKAGE_NAME} == "client" ]; then
find "${OUTPUT_DIR}" -path "${OUTPUT_DIR}/base" -prune -o -type f -a -name \*.md -exec sed -i 's/kubernetes_asyncio.client-python/client-python/g' {} +

# workaround https://github.com/swagger-api/swagger-codegen/pull/7905
# TODO: Remove this when above merges
find "${OUTPUT_DIR}/client" -type f -name \*.py ! -name '__init__.py' -exec sed -i '/^from .*models.*/d' {} \;

# workaround https://github.com/swagger-api/swagger-codegen/pull/8204
# + closing session
# + support application/strategic-merge-patch+json
# workaround https://github.com/swagger-api/swagger-codegen/pull/8797
# + aiohttp without verify_ssl
# TODO: Remove this when above merges
patch "${OUTPUT_DIR}/client/rest.py" "${SCRIPT_ROOT}/python-asyncio-rest.py.patch"

# workaround https://github.com/swagger-api/swagger-codegen/pull/8401
# TODO: Remove this when above merges

find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i 's/async=/async_req=/g' {} +
find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i 's/async bool/async_req bool/g' {} +
find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i "s/'async'/'async_req'/g" {} +
find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i "s/async parameter/async_req parameter/g" {} +
find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i "s/if not async/if not async_req/g" {} +

# workaround https://github.com/swagger-api/swagger-codegen/pull/8061 (thread pool only on demand)
# TODO: Remove this when above merges
patch "${OUTPUT_DIR}/client/api_client.py" "${SCRIPT_ROOT}/python-asyncio-api_client.py.patch"

# fix imports
find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i 's/import client\./import kubernetes_asyncio.client./g' {} +
find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i 's/from client/from kubernetes_asyncio.client/g' {} +
Expand Down
5 changes: 5 additions & 0 deletions openapi/python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,17 @@ kubeclient::generator::generate_client "${OUTPUT_DIR}"
echo "--- Patching generated code..."

# workaround https://github.com/swagger-api/swagger-codegen/pull/8401
# TODO: Remove this when above merges
find "${OUTPUT_DIR}/${PACKAGE_NAME}/" -type f -name \*.py -exec sed -i 's/async=/async_req=/g' {} +
find "${OUTPUT_DIR}/${PACKAGE_NAME}/" -type f -name \*.py -exec sed -i 's/async bool/async_req bool/g' {} +
find "${OUTPUT_DIR}/${PACKAGE_NAME}/" -type f -name \*.py -exec sed -i "s/'async'/'async_req'/g" {} +
sed -i "s/if not async/if not async_req/g" "${OUTPUT_DIR}/${PACKAGE_NAME}/api_client.py"
#

# workaround https://github.com/swagger-api/swagger-codegen/pull/8061 (thread pool only on demand)
# TODO: Remove this when above merges
patch "${OUTPUT_DIR}/${PACKAGE_NAME}/api_client.py" "${SCRIPT_ROOT}/python-api_client.py.patch"

find "${OUTPUT_DIR}/test" -type f -name \*.py -exec sed -i 's/\bclient/kubernetes.client/g' {} +
find "${OUTPUT_DIR}" -path "${OUTPUT_DIR}/base" -prune -o -type f -a -name \*.md -exec sed -i 's/\bclient/kubernetes.client/g' {} +
find "${OUTPUT_DIR}" -path "${OUTPUT_DIR}/base" -prune -o -type f -a -name \*.md -exec sed -i 's/kubernetes.client-python/client-python/g' {} +
Expand Down

0 comments on commit 9d14077

Please sign in to comment.