From 99bb0924918c65c213b17521528377f8aa033b7c Mon Sep 17 00:00:00 2001 From: XuHugo Date: Sat, 23 Jan 2021 08:58:38 +0800 Subject: [PATCH 1/8] Modify models modify models *Make changes on the basis of the original and try to keep the original *add zip function Signed-off-by: XuHugo --- src/api-engine/api/models.py | 256 +++++++++++++++++---------- src/api-engine/api/utils/__init__.py | 1 + src/api-engine/api/utils/common.py | 30 ++++ 3 files changed, 191 insertions(+), 96 deletions(-) diff --git a/src/api-engine/api/models.py b/src/api-engine/api/models.py index e4525dce8..2e3c01011 100644 --- a/src/api-engine/api/models.py +++ b/src/api-engine/api/models.py @@ -15,6 +15,7 @@ from django.db import models from django.dispatch import receiver from django.db.models.signals import post_save +from django.contrib.postgres.fields import ArrayField from api.common.enums import ( HostStatus, @@ -71,6 +72,30 @@ class Organization(models.Model): default="", max_length=64, help_text="Name of organization" ) created_at = models.DateTimeField(auto_now_add=True) + msp = models.TextField(help_text="msp of organization", null=True) + tls = models.TextField(help_text="tls of organization", null=True) + agents = models.CharField( + help_text="agent of organization", + max_length=128, + default="", + ) + network = models.ForeignKey( + "Network", + help_text="Network to which the organization belongs", + null=True, + related_name="network", + on_delete=models.SET_NULL + ) + channel = models.ForeignKey( + "Channel", + help_text="channel to which the organization belongs", + null=True, + related_name="channel", + on_delete=models.SET_NULL + ) + + class Meta: + ordering = ("-created_at",) class UserProfile(AbstractUser): @@ -137,21 +162,17 @@ class Agent(models.Model): max_length=64, default=random_name("agent"), ) - image = models.CharField( - help_text="Image name for deploy agent", max_length=64, default="" - ) - ip = models.GenericIPAddressField(help_text="Agent IP Address") - govern = models.ForeignKey( - Govern, - help_text="Govern of agent", + urls = models.URLField( + help_text="Agent URL", null=True, - on_delete=models.CASCADE, + blank=True ) organization = models.ForeignKey( "Organization", null=True, on_delete=models.CASCADE, help_text="Organization of agent", + related_name="organization", ) status = models.CharField( help_text="Status of agent", @@ -159,34 +180,12 @@ class Agent(models.Model): max_length=10, default=HostStatus.Active.name.lower(), ) - log_level = models.CharField( - help_text="Log level of agent", - choices=LogLevel.to_choices(True), - max_length=10, - default=LogLevel.Info.name.lower(), - ) type = models.CharField( help_text="Type of agent", choices=HostType.to_choices(True), max_length=32, default=HostType.Docker.name.lower(), ) - schedulable = models.BooleanField( - help_text="Whether agent can be scheduled", default=True - ) - capacity = models.IntegerField( - help_text="Capacity of agent", - default=1, - validators=[MinValueValidator(1), MaxValueValidator(MAX_CAPACITY)], - ) - node_capacity = models.IntegerField( - help_text="Capacity of node", - default=6, - validators=[ - MinValueValidator(1), - MaxValueValidator(MAX_NODE_CAPACITY), - ], - ) config_file = models.FileField( help_text="Config file for agent", max_length=256, @@ -284,8 +283,10 @@ class Network(models.Model): default=make_uuid, editable=True, ) - govern = models.ForeignKey( - Govern, help_text="Govern of node", null=True, on_delete=models.CASCADE + name = models.CharField( + help_text="network name, can be generated automatically.", + max_length=64, + default=random_name("netowrk"), ) type = models.CharField( help_text="Type of network, %s" % NetworkType.values(), @@ -304,6 +305,12 @@ class Network(models.Model): created_at = models.DateTimeField( help_text="Create time of network", auto_now_add=True ) + consensus = models.CharField( + help_text="Consensus of network", max_length=128, default="raft", + ) + organizations = ArrayField( + models.CharField(max_length=128, blank=True), help_text="organizations of network", default=list, null=True + ) class Meta: ordering = ("-created_at",) @@ -449,21 +456,6 @@ class Node(models.Model): editable=True, ) name = models.CharField(help_text="Node name", max_length=64, default="") - network_type = models.CharField( - help_text="Network type of node", - choices=NetworkType.to_choices(), - default=NetworkType.Fabric.value, - max_length=64, - ) - network_version = models.CharField( - help_text=""" - Version of network for node. - Fabric supported versions: %s - """ - % (FabricVersions.values()), - max_length=64, - default="", - ) type = models.CharField( help_text=""" Node type defined for network. @@ -472,18 +464,6 @@ class Node(models.Model): % (FabricNodeType.names()), max_length=64, ) - ca = models.ForeignKey( - FabricCA, - help_text="CA configuration of node", - null=True, - on_delete=models.CASCADE, - ) - peer = models.ForeignKey( - FabricPeer, - help_text="Peer configuration of node", - null=True, - on_delete=models.CASCADE, - ) urls = JSONField( help_text="URL configurations for node", null=True, @@ -496,24 +476,26 @@ class Node(models.Model): null=True, on_delete=models.CASCADE, ) - govern = models.ForeignKey( - Govern, help_text="Govern of node", null=True, on_delete=models.CASCADE - ) - organization = models.ForeignKey( + org = models.ForeignKey( Organization, help_text="Organization of node", null=True, + related_name="org", on_delete=models.CASCADE, ) - agent = models.ForeignKey( - Agent, help_text="Agent of node", null=True, on_delete=models.CASCADE - ) - network = models.ForeignKey( - Network, - help_text="Network which node joined.", - on_delete=models.CASCADE, - null=True, - ) + # agent = models.ForeignKey( + # Agent, + # help_text="Agent of node", + # null=True, + # related_name="network", + # on_delete=models.CASCADE + # ) + # network = models.ForeignKey( + # Network, + # help_text="Network which node joined.", + # on_delete=models.CASCADE, + # null=True, + # ) created_at = models.DateTimeField( help_text="Create time of network", auto_now_add=True ) @@ -523,18 +505,18 @@ class Node(models.Model): max_length=64, default=NodeStatus.Deploying.name.lower(), ) - compose_file = models.FileField( - help_text="Compose file for node, if agent type is docker.", + config_file = models.TextField( + help_text="Config file of node", max_length=256, - upload_to=get_compose_file_path, blank=True, null=True, ) - file = models.FileField( - help_text="File of node", - max_length=256, - blank=True, - upload_to=get_node_file_path, + msp = models.TextField( + help_text="msp of node", + null=True, + ) + tls = models.TextField( + help_text="tls of node", null=True, ) @@ -561,22 +543,22 @@ def save( force_insert, force_update, using, update_fields ) - def delete(self, using=None, keep_parents=False): - if self.compose_file: - compose_file_path = Path(self.compose_file.path) - if os.path.isdir(os.path.dirname(compose_file_path)): - shutil.rmtree(os.path.dirname(compose_file_path)) - - # remove related files of node - if self.file: - file_path = Path(self.file.path) - if os.path.isdir(os.path.dirname(file_path)): - shutil.rmtree(os.path.dirname(file_path)) - - if self.ca: - self.ca.delete() - - super(Node, self).delete(using, keep_parents) + # def delete(self, using=None, keep_parents=False): + # if self.compose_file: + # compose_file_path = Path(self.compose_file.path) + # if os.path.isdir(os.path.dirname(compose_file_path)): + # shutil.rmtree(os.path.dirname(compose_file_path)) + # + # # remove related files of node + # if self.file: + # file_path = Path(self.file.path) + # if os.path.isdir(os.path.dirname(file_path)): + # shutil.rmtree(os.path.dirname(file_path)) + # + # if self.ca: + # self.ca.delete() + # + # super(Node, self).delete(using, keep_parents) class NodeUser(models.Model): @@ -684,3 +666,85 @@ class File(models.Model): class Meta: ordering = ("-created_at",) + + class User(models.Model): + id = models.UUIDField( + primary_key=True, + help_text="ID of user", + default=make_uuid, + editable=True, + ) + name = models.CharField( + help_text="user name", max_length=128 + ) + roles = models.CharField( + help_text="roles of user", max_length=128 + ) + organization = models.ForeignKey("Organization", on_delete=models.CASCADE) + attributes = models.CharField( + help_text="attributes of user", max_length=128 + ) + revoked = models.CharField( + help_text="revoked of user", max_length=128 + ) + create_ts = models.DateTimeField( + help_text="Create time of user", auto_now_add=True + ) + msp = models.TextField( + help_text="msp of user", + null=True, + ) + tls = models.TextField( + help_text="tls of user", + null=True, + ) + + class Channel(models.Model): + id = models.UUIDField( + primary_key=True, + help_text="ID of Channel", + default=make_uuid, + editable=False, + unique=True) + name = models.CharField( + help_text="name of channel", max_length=128 + ) + network = models.ForeignKey( + "Network", on_delete=models.CASCADE + ) + create_ts = models.DateTimeField( + help_text="Create time of Channel", auto_now_add=True + ) + + class ChainCode(models.Model): + id = models.UUIDField( + primary_key=True, + help_text="ID of chainCode", + default=make_uuid, + editable=False, + unique=True + ) + name = models.CharField( + help_text="ChainCode name", max_length=128 + ) + version = models.CharField( + help_text="version of chainCode", max_length=128 + ) + creator = models.CharField( + help_text="creator of chainCode", max_length=128 + ) + language = models.CharField( + help_text="language of chainCode", max_length=128 + ) + channel = models.ManyToManyField("Channel") + install_times = models.DateTimeField( + help_text="Create time of install", auto_now_add=True + ) + instantiate_times = models.DateTimeField( + help_text="Create time of instantiate", auto_now_add=True + ) + node = models.ManyToManyField("Node", related_name='node') + status = models.CharField( + help_text="status of chainCode", max_length=128 + ) + diff --git a/src/api-engine/api/utils/__init__.py b/src/api-engine/api/utils/__init__.py index 8710d97a0..9b0d07bf1 100644 --- a/src/api-engine/api/utils/__init__.py +++ b/src/api-engine/api/utils/__init__.py @@ -8,6 +8,7 @@ from api.common.enums import ErrorCode from rest_framework import status from rest_framework.exceptions import ErrorDetail +from .common import zip_dir, zip_file LOG = logging.getLogger(__name__) diff --git a/src/api-engine/api/utils/common.py b/src/api-engine/api/utils/common.py index 35dbc5c38..e182f84f3 100644 --- a/src/api-engine/api/utils/common.py +++ b/src/api-engine/api/utils/common.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # import hashlib +import os from drf_yasg import openapi from rest_framework import status @@ -10,6 +11,7 @@ from functools import reduce, partial from api.common.serializers import BadResponseSerializer import uuid +from zipfile import ZipFile def make_uuid(): @@ -99,3 +101,31 @@ def hash_file(file, block_size=65536): hash_func.update(buf) return hash_func.hexdigest() + + +def zip_dir(dirpath, outFullName): + """ + Compress the specified folder + :param dirpath: specified folder + :param outFullName: Save path+xxxx.zip + :return: null + """ + dir_dst = "/" + dirpath.rsplit("/", 1)[1] + zdir = ZipFile(outFullName, "w") + for path, dirnames, filenames in os.walk(dirpath): + fpath = dir_dst + path.replace(dirpath, '') + for filename in filenames: + zdir.write(os.path.join(path, filename), os.path.join(fpath, filename)) + zdir.close() + + +def zip_file(dirpath, outFullName): + """ + Compress the specified file + :param dirpath: specified folder of file + :param outFullName: Save path+filename.zip + :return: null + """ + zfile = ZipFile(outFullName, "w") + zfile.write(dirpath, dirpath.rsplit("/", 1)[1]) + zfile.close() From beef4a0654cb4b8254dc6a4a9523600625dd05ee Mon Sep 17 00:00:00 2001 From: XuHugo Date: Sat, 23 Jan 2021 09:16:18 +0800 Subject: [PATCH 2/8] Modify organization Modify organization: *add config.py *modify organization Signed-off-by: XuHugo --- src/api-engine/api/config.py | 9 +++ .../api/routes/organization/views.py | 71 ++++++++++++++++--- 2 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 src/api-engine/api/config.py diff --git a/src/api-engine/api/config.py b/src/api-engine/api/config.py new file mode 100644 index 000000000..be798ed7c --- /dev/null +++ b/src/api-engine/api/config.py @@ -0,0 +1,9 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +CELLO_HOME = "/opt/cello" +FABRIC_TOOL = "/opt/bin" + +FABRIC_PEER_CFG = "/opt/node/peer.yaml.bak" +FABRIC_ORDERER_CFG = "/opt/node/orderer.yaml.bak" +FABRIC_CA_CFG = "/opt/node/ca.yaml.bak" \ No newline at end of file diff --git a/src/api-engine/api/routes/organization/views.py b/src/api-engine/api/routes/organization/views.py index dbcbb72dc..672412373 100644 --- a/src/api-engine/api/routes/organization/views.py +++ b/src/api-engine/api/routes/organization/views.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # import logging +import base64 from rest_framework import viewsets, status from rest_framework.decorators import action @@ -25,13 +26,18 @@ from api.routes.user.serializers import UserIDSerializer from api.models import UserProfile, Organization from api.routes.user.serializers import UserListSerializer, UserQuerySerializer +from api.lib.pki import CryptoGen, CryptoConfig +from api.utils import zip_dir, zip_file +from api.config import CELLO_HOME + LOG = logging.getLogger(__name__) class OrganizationViewSet(viewsets.ViewSet): - authentication_classes = (JSONWebTokenAuthentication,) - permission_classes = (IsAuthenticated, IsOperatorAuthenticated) + """Class represents orgnization related operations.""" + #authentication_classes = (JSONWebTokenAuthentication,) + #permission_classes = (IsAuthenticated, IsOperatorAuthenticated) @swagger_auto_schema( query_serializer=OrganizationQuery, @@ -43,7 +49,9 @@ def list(self, request): """ List Organizations - List organizations through query parameter + :param request: query parameter + :return: organization list + :rtype: list """ serializer = OrganizationQuery(data=request.GET) if serializer.is_valid(raise_exception=True): @@ -82,7 +90,9 @@ def create(self, request): """ Create Organization - Create Organization + :param request: create parameter + :return: organization ID + :rtype: uuid """ serializer = OrganizationCreateBody(data=request.data) if serializer.is_valid(raise_exception=True): @@ -93,7 +103,13 @@ def create(self, request): pass else: raise ResourceExists - organization = Organization(name=name) + + CryptoConfig(name).create() + CryptoGen(name).generate() + + msp, tls = self._conversion_msp_tls(name) + + organization = Organization(name=name, msp=msp, tls=tls) organization.save() response = OrganizationIDSerializer(data=organization.__dict__) @@ -102,6 +118,30 @@ def create(self, request): response.validated_data, status=status.HTTP_201_CREATED ) + def _conversion_msp_tls(self, name): + """ + msp and tls from zip file to byte + + :param name: organization name + :return: msp, tls + :rtype: bytes + """ + try: + dir_org = "{}/{}/crypto-config/peerOrganizations/{}/" \ + .format(CELLO_HOME, name, name) + + zip_dir("{}msp".format(dir_org), "{}msp.zip".format(dir_org)) + with open("{}msp.zip".format(dir_org), "rb") as f_msp: + msp = base64.b64encode(f_msp.read()) + + zip_dir("{}tlsca".format(dir_org), "{}tls.zip".format(dir_org)) + with open("{}tls.zip".format(dir_org), "rb") as f_tls: + tls = base64.b64encode(f_tls.read()) + except Exception as e: + raise e + + return msp, tls + @swagger_auto_schema( responses=with_common_response( {status.HTTP_204_NO_CONTENT: "No Content"} @@ -111,15 +151,21 @@ def destroy(self, request, pk=None): """ Delete Organization - Delete Organization + :param request: destory parameter + :param pk: primary key + :return: none + :rtype: rest_framework.status """ try: organization = Organization.objects.get(id=pk) - user_count = UserProfile.objects.filter( - organization=organization - ).count() - if user_count > 0: + if organization.network: raise ResourceInUse + + # user_count = UserProfile.objects.filter( + # organization=organization + # ).count() + # if user_count > 0: + # raise ResourceInUse organization.delete() except ObjectDoesNotExist: raise ResourceNotFound @@ -135,7 +181,10 @@ def retrieve(self, request, pk=None): """ Retrieve Organization - Retrieve Organization + :param request: retrieve parameter + :param pk: primary key + :return: organization info + :rtype: OrganizationResponse """ try: organization = Organization.objects.get(id=pk) From 22be3655c1500d020a1c398cb2c3ebf8fb168760 Mon Sep 17 00:00:00 2001 From: XuHugo Date: Sat, 23 Jan 2021 09:22:35 +0800 Subject: [PATCH 3/8] Modify node modify node: *modify the view of node Signed-off-by: XuHugo --- src/api-engine/api/routes/node/serializers.py | 93 ++++---- src/api-engine/api/routes/node/views.py | 221 ++++++++---------- 2 files changed, 136 insertions(+), 178 deletions(-) diff --git a/src/api-engine/api/routes/node/serializers.py b/src/api-engine/api/routes/node/serializers.py index 3dd8609b2..11588f660 100644 --- a/src/api-engine/api/routes/node/serializers.py +++ b/src/api-engine/api/routes/node/serializers.py @@ -42,8 +42,6 @@ class Meta: "per_page", "type", "name", - "network_type", - "network_version", "agent_id", ) extra_kwargs = {"type": {"required": False}} @@ -238,69 +236,56 @@ class Meta: class NodeCreateBody(serializers.ModelSerializer): - agent_type = serializers.ChoiceField( - help_text="Agent type", - choices=HostType.to_choices(True), - required=False, - ) - ca = FabricCASerializer( - help_text="CA configuration for node", required=False - ) - peer = FabricPeerSerializer( - help_text="Peer configuration for node", required=False - ) + organization = serializers.UUIDField(help_text="ID of organization") class Meta: model = Node fields = ( - "network_type", - "network_version", + "name", + "urls", "type", - "agent_type", - "agent", - "ca", - "peer", + "organization", ) extra_kwargs = { - "network_type": {"required": True}, - "network_version": {"required": True}, + "name": {"required": True}, + "urls": {"required": True}, "type": {"required": True}, } def validate(self, attrs): - network_type = attrs.get("network_type") - node_type = attrs.get("type") - network_version = attrs.get("network_version") - agent_type = attrs.get("agent_type") - agent = attrs.get("agent") - ca = attrs.get("ca") - peer = attrs.get("peer") - if network_type == NetworkType.Fabric.value: - if network_version not in FabricVersions.values(): - raise serializers.ValidationError("Not valid fabric version") - if node_type not in FabricNodeType.names(): - raise serializers.ValidationError( - "Not valid node type for %s" % network_type - ) - if node_type == FabricNodeType.Ca.name.lower() and ca is None: - raise serializers.ValidationError( - "Please input ca configuration for ca node" - ) - elif ( - node_type == FabricNodeType.Peer.name.lower() and peer is None - ): - raise serializers.ValidationError( - "Please input peer configuration for peer node" - ) - - if agent_type is None and agent is None: - raise serializers.ValidationError("Please set agent_type or agent") - - if agent_type and agent: - if agent_type != agent.type: - raise serializers.ValidationError( - "agent type not equal to agent" - ) + # network_type = attrs.get("network_type") + # node_type = attrs.get("type") + # network_version = attrs.get("network_version") + # agent_type = attrs.get("agent_type") + # agent = attrs.get("agent") + # ca = attrs.get("ca") + # peer = attrs.get("peer") + # if network_type == NetworkType.Fabric.value: + # if network_version not in FabricVersions.values(): + # raise serializers.ValidationError("Not valid fabric version") + # if node_type not in FabricNodeType.names(): + # raise serializers.ValidationError( + # "Not valid node type for %s" % network_type + # ) + # if node_type == FabricNodeType.Ca.name.lower() and ca is None: + # raise serializers.ValidationError( + # "Please input ca configuration for ca node" + # ) + # elif ( + # node_type == FabricNodeType.Peer.name.lower() and peer is None + # ): + # raise serializers.ValidationError( + # "Please input peer configuration for peer node" + # ) + # + # if agent_type is None and agent is None: + # raise serializers.ValidationError("Please set agent_type or agent") + # + # if agent_type and agent: + # if agent_type != agent.type: + # raise serializers.ValidationError( + # "agent type not equal to agent" + # ) return attrs diff --git a/src/api-engine/api/routes/node/views.py b/src/api-engine/api/routes/node/views.py index da6885737..775459e03 100644 --- a/src/api-engine/api/routes/node/views.py +++ b/src/api-engine/api/routes/node/views.py @@ -3,6 +3,7 @@ # import json import logging +import base64 from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import PermissionDenied @@ -23,6 +24,7 @@ from api.models import ( Agent, Node, + Organization, Port, FabricCA, FabricNodeType, @@ -50,21 +52,25 @@ from api.tasks import operate_node from api.utils.common import with_common_response from api.auth import CustomAuthenticate +from api.lib.pki import CryptoGen, CryptoConfig +from api.utils import zip_dir, zip_file +from api.config import CELLO_HOME + LOG = logging.getLogger(__name__) class NodeViewSet(viewsets.ViewSet): - authentication_classes = (CustomAuthenticate, JSONWebTokenAuthentication) + #authentication_classes = (CustomAuthenticate, JSONWebTokenAuthentication) # Only operator can update node info - def get_permissions(self): - if self.action in ["update"]: - permission_classes = (IsAuthenticated, IsOperatorAuthenticated) - else: - permission_classes = (IsAuthenticated,) - - return [permission() for permission in permission_classes] + # def get_permissions(self): + # if self.action in ["update"]: + # permission_classes = (IsAuthenticated, IsOperatorAuthenticated) + # else: + # permission_classes = (IsAuthenticated,) + # + # return [permission() for permission in permission_classes] @staticmethod def _validate_organization(request): @@ -79,9 +85,11 @@ def _validate_organization(request): ) def list(self, request, *args, **kwargs): """ - List Nodes + List node - Filter nodes with query parameters. + :param request: query parameter + :return: node list + :rtype: list """ serializer = NodeQuery(data=request.GET) if serializer.is_valid(raise_exception=True): @@ -89,28 +97,22 @@ def list(self, request, *args, **kwargs): per_page = serializer.validated_data.get("per_page") node_type = serializer.validated_data.get("type") name = serializer.validated_data.get("name") - network_type = serializer.validated_data.get("network_type") - network_version = serializer.validated_data.get("network_version") agent_id = serializer.validated_data.get("agent_id") - if agent_id is not None and not request.user.is_operator: - raise PermissionDenied + # if agent_id is not None and not request.user.is_operator: + # raise PermissionDenied query_filter = {} if node_type: query_filter.update({"type": node_type}) if name: query_filter.update({"name__icontains": name}) - if network_type: - query_filter.update({"network_type": network_type}) - if network_version: - query_filter.update({"network_version": network_version}) - if request.user.is_administrator: - query_filter.update( - {"organization": request.user.organization} - ) - elif request.user.is_common_user: - query_filter.update({"user": request.user}) + # if request.user.is_administrator: + # query_filter.update( + # {"organization": request.user.organization} + # ) + # elif request.user.is_common_user: + # query_filter.update({"user": request.user}) if agent_id: query_filter.update({"agent__id": agent_id}) nodes = Node.objects.filter(**query_filter) @@ -254,81 +256,76 @@ def create(self, request): """ Create Node - Create node + :param request: create parameter + :return: node ID + :rtype: uuid """ serializer = NodeCreateBody(data=request.data) if serializer.is_valid(raise_exception=True): - self._validate_organization(request) - agent_type = serializer.validated_data.get("agent_type") - network_type = serializer.validated_data.get("network_type") - network_version = serializer.validated_data.get("network_version") - agent = serializer.validated_data.get("agent") - node_type = serializer.validated_data.get("type") - ca = serializer.validated_data.get("ca") - peer = serializer.validated_data.get("peer") - if agent is None: - available_agents = ( - Agent.objects.annotate(network_num=Count("node__network")) - .annotate(node_num=Count("node")) - .filter( - schedulable=True, - type=agent_type, - network_num__lt=F("capacity"), - node_num__lt=F("node_capacity"), - organization=request.user.organization, - ) - .order_by("node_num") - ) - if len(available_agents) > 0: - agent = available_agents[0] - else: - raise NoResource + #self._validate_organization(request) + name = serializer.validated_data.get("name") + type = serializer.validated_data.get("type") + urls = serializer.validated_data.get("urls") + organization = serializer.validated_data.get("organization") + + org = Organization.objects.get(id=organization) + if org.organization.all(): + pass else: - if not request.user.is_operator: - raise PermissionDenied - node_count = Node.objects.filter(agent=agent).count() - if node_count >= agent.node_capacity or not agent.schedulable: - raise NoResource - - fabric_ca = None - fabric_peer = None - peer_ca_list = [] - if node_type == FabricNodeType.Ca.name.lower(): - fabric_ca = self._save_fabric_ca(request, ca) - elif node_type == FabricNodeType.Peer.name.lower(): - fabric_peer, peer_ca_list = self._save_fabric_peer( - request, peer - ) + raise NoResource + nodes = { + "type": type, + "Specs": [name] + } + CryptoConfig(org.name).update(nodes) + CryptoGen(org.name).extend() + + msp, tls = self._conversion_msp_tls(type, org.name, name) + node = Node( - network_type=network_type, - agent=agent, - network_version=network_version, - user=request.user, - organization=request.user.organization, - type=node_type, - ca=fabric_ca, - peer=fabric_peer, + name=name, + org=org, + urls=urls, + type=type, + msp=msp, + tls=tls ) node.save() - agent_config_file = request.build_absolute_uri( - agent.config_file.url - ) - node_detail_url = reverse("node-detail", args=[str(node.id)]) - node_detail_url = request.build_absolute_uri(node_detail_url) - node_file_upload_api = reverse("node-files", args=[str(node.id)]) - node_file_upload_api = request.build_absolute_uri( - node_file_upload_api - ) - operate_node.delay( - str(node.id), - AgentOperation.Create.value, - agent_config_file=agent_config_file, - node_detail_url=node_detail_url, - node_file_upload_api=node_file_upload_api, - peer_ca_list=json.dumps(peer_ca_list), - ) - response = NodeIDSerializer({"id": str(node.id)}) - return Response(response.data, status=status.HTTP_201_CREATED) + + response = NodeIDSerializer(data=node.__dict__) + if response.is_valid(raise_exception=True): + return Response( + response.validated_data, status=status.HTTP_201_CREATED + ) + + def _conversion_msp_tls(self, type, org, node): + """ + msp and tls from zip file to byte + + :param org: organization name + :param type: node type + :param node: node name + :return: msp, tls + :rtype: bytes + """ + try: + if type == "peer": + dir_node = "{}/{}/crypto-config/peerOrganizations/{}/peers/{}/" \ + .format(CELLO_HOME, org, org, node + "." + org) + else: + dir_node = "{}/{}/crypto-config/ordererOrganizations/{}/orderers/{}/" \ + .format(CELLO_HOME, org, org.split(".", 1)[1], node + "." + org.split(".", 1)[1]) + zip_dir("{}msp".format(dir_node), "{}msp.zip".format(dir_node)) + with open("{}msp.zip".format(dir_node), "rb") as f_msp: + msp = base64.b64encode(f_msp.read()) + + zip_dir("{}tls".format(dir_node), "{}tls.zip".format(dir_node)) + with open("{}tls.zip".format(dir_node), "rb") as f_tls: + tls = base64.b64encode(f_tls.read()) + except Exception as e: + raise e + + return msp, tls @swagger_auto_schema( methods=["post"], @@ -353,43 +350,19 @@ def destroy(self, request, pk=None): """ Delete Node - Delete node + :param request: destory parameter + :param pk: primary key + :return: none + :rtype: rest_framework.status """ try: - if request.user.is_superuser: - node = Node.objects.get(id=pk) - else: - node = Node.objects.get( - id=pk, organization=request.user.organization - ) + node = Node.objects.get(id=pk) + node.delete() + # todo delete node from agent except ObjectDoesNotExist: raise ResourceNotFound - else: - if node.status != NodeStatus.Deleting.name.lower(): - if node.status not in [ - NodeStatus.Error.name.lower(), - NodeStatus.Deleted.name.lower(), - ]: - node.status = NodeStatus.Deleting.name.lower() - node.save() - - agent_config_file = request.build_absolute_uri( - node.agent.config_file.url - ) - node_detail_url = reverse("node-detail", args=[pk]) - node_detail_url = request.build_absolute_uri( - node_detail_url - ) - operate_node.delay( - str(node.id), - AgentOperation.Delete.value, - agent_config_file=agent_config_file, - node_detail_url=node_detail_url, - ) - else: - node.delete() - return Response(status=status.HTTP_204_NO_CONTENT) + return Response(status=status.HTTP_204_NO_CONTENT) @swagger_auto_schema( operation_id="update node", From b9a4c7cd9b2cf82e7e8f1ff7f0d3e921fa3e072b Mon Sep 17 00:00:00 2001 From: XuHugo Date: Sat, 23 Jan 2021 09:27:03 +0800 Subject: [PATCH 4/8] Modify network Modify network: *modify network Signed-off-by: XuHugo --- .../api/routes/network/serializers.py | 35 +++---- src/api-engine/api/routes/network/views.py | 94 +++++++++++++++++-- 2 files changed, 107 insertions(+), 22 deletions(-) diff --git a/src/api-engine/api/routes/network/serializers.py b/src/api-engine/api/routes/network/serializers.py index 03cd6df4e..7c70258ba 100644 --- a/src/api-engine/api/routes/network/serializers.py +++ b/src/api-engine/api/routes/network/serializers.py @@ -31,6 +31,10 @@ class NetworkQuery(serializers.Serializer): help_text=NetworkStatus.get_info("Network Status:", list_str=True), choices=NetworkStatus.to_choices(True), ) + class Meta: + model = Network + fields = ("page", "per_page", "name") + extra_kwargs = {"name": {"required": False}} class NetworkIDSerializer(serializers.Serializer): @@ -38,12 +42,18 @@ class NetworkIDSerializer(serializers.Serializer): class NetworkResponse(NetworkIDSerializer): - status = serializers.ChoiceField( - help_text=NetworkStatus.get_info("Network Status:", list_str=True), - choices=NetworkStatus.to_choices(True), - ) + id = serializers.UUIDField(help_text="ID of Network") + created_at = serializers.DateTimeField(help_text="Network create time") - updated_at = serializers.DateTimeField(help_text="Network update time") + + class Meta: + model = Network + fields = ("id", "name", "created_at") + extra_kwargs = { + "name": {"required": True}, + "created_at": {"required": True, "read_only": False}, + "id": {"required": True, "read_only": False}, + } class NetworkMemberSerializer(serializers.Serializer): @@ -56,20 +66,13 @@ class NetworkMemberSerializer(serializers.Serializer): class NetworkCreateBody(serializers.ModelSerializer): - method = serializers.ChoiceField( - help_text=NetworkCreateType.get_info( - "Network Create Types:", list_str=True - ), - choices=NetworkCreateType.to_choices(True), - ) class Meta: model = Network - fields = ("type", "version", "method") - extra_kwargs = { - "type": {"required": True}, - "version": {"required": True}, - } + fields = ("name", "consensus", "organizations") + extra_kwargs = {"name": {"required": True}, + "consensus": {"required": True}, + "organizations": {"required": True}} class NetworkMemberResponse(serializers.Serializer): diff --git a/src/api-engine/api/routes/network/views.py b/src/api-engine/api/routes/network/views.py index edcad9310..b26aca2f0 100644 --- a/src/api-engine/api/routes/network/views.py +++ b/src/api-engine/api/routes/network/views.py @@ -7,6 +7,8 @@ from rest_framework.decorators import action from rest_framework.response import Response from drf_yasg.utils import swagger_auto_schema +from django.core.paginator import Paginator +from django.core.exceptions import ObjectDoesNotExist from api.routes.network.serializers import ( NetworkQuery, NetworkListResponse, @@ -19,6 +21,8 @@ NetworkIDSerializer, ) from api.utils.common import with_common_response +from api.lib.configtxgen import ConfigTX, ConfigTxGen +from api.models import Network, Node, Organization LOG = logging.getLogger(__name__) @@ -32,10 +36,38 @@ class NetworkViewSet(viewsets.ViewSet): ) def list(self, request): """ - List Networks + List network - Filter networks with query parameters. + :param request: query parameter + :return: network list + :rtype: list """ + serializer = NetworkQuery(data=request.GET) + if serializer.is_valid(raise_exception=True): + page = serializer.validated_data.get("page", 1) + per_page = serializer.validated_data.get("per_page", 10) + name = serializer.validated_data.get("name") + parameters = {} + if name: + parameters.update({"name__icontains": name}) + networks = Network.objects.filter(**parameters) + p = Paginator(networks, per_page) + networks = p.page(page) + networks = [ + { + "id": str(network.id), + "name": network.name, + "created_at": network.created_at, + } + for network in networks + ] + response = NetworkListResponse( + data={"total": p.count, "data": networks} + ) + if response.is_valid(raise_exception=True): + return Response( + response.validated_data, status=status.HTTP_200_OK + ) return Response(data=[], status=status.HTTP_200_OK) @swagger_auto_schema( @@ -46,12 +78,48 @@ def list(self, request): ) def create(self, request): """ - New Network + Create Network - Create new network through internal nodes, - or import exists network outside + :param request: create parameter + :return: organization ID + :rtype: uuid """ - pass + serializer = NetworkCreateBody(data=request.data) + if serializer.is_valid(raise_exception=True): + name = serializer.validated_data.get("name") + consensus = serializer.validated_data.get("consensus") + organizations = serializer.validated_data.get("organizations") + + try: + Network.objects.get(name=name) + except ObjectDoesNotExist: + pass + orderers = [] + peers = [] + i = 0 + for organization in organizations: + org = Organization.objects.get(pk=organization) + orderers.append({"name": org.name, "hosts": []}) + peers.append({"name": org.name, "hosts": []}) + nodes = Node.objects.filter(org=org) + for node in nodes: + if node.type == "peer": + peers[i]["hosts"].append({"name": node.name, "port": node.urls.split(":")[2]}) + elif node.type == "orderer": + orderers[i]["hosts"].append({"name": node.name, "port": node.urls.split(":")[2]}) + i = i + 1 + + ConfigTX(name).create(consensus=consensus, orderers=orderers, peers=peers) + ConfigTxGen(name).genesis() + + network = Network(name=name, consensus=consensus, organizations=organizations) + network.save() + + response = NetworkIDSerializer(data=network.__dict__) + if response.is_valid(raise_exception=True): + return Response( + response.validated_data, status=status.HTTP_201_CREATED + ) @swagger_auto_schema(responses=with_common_response()) def retrieve(self, request, pk=None): @@ -62,6 +130,20 @@ def retrieve(self, request, pk=None): """ pass + @swagger_auto_schema( + responses=with_common_response( + {status.HTTP_204_NO_CONTENT: "No Content"} + ) + ) + def destroy(self, request, pk=None): + try: + network = Network.objects.get(pk=pk) + network.delete() + except ObjectDoesNotExist: + raise BaseException + + return Response(status=status.HTTP_204_NO_CONTENT) + @swagger_auto_schema( methods=["get"], responses=with_common_response( From be5b6382570b28c26c602c1a5b4ffbad3e2e43ff Mon Sep 17 00:00:00 2001 From: XuHugo Date: Sat, 23 Jan 2021 09:31:49 +0800 Subject: [PATCH 5/8] Modify agent Modify agent: *modify agent Signed-off-by: XuHugo --- .../api/routes/agent/serializers.py | 48 ++++-------- src/api-engine/api/routes/agent/views.py | 77 +++++++++---------- 2 files changed, 49 insertions(+), 76 deletions(-) diff --git a/src/api-engine/api/routes/agent/serializers.py b/src/api-engine/api/routes/agent/serializers.py index cf3bd3e5b..050963d8b 100644 --- a/src/api-engine/api/routes/agent/serializers.py +++ b/src/api-engine/api/routes/agent/serializers.py @@ -109,6 +109,8 @@ def validate(self, attrs): class AgentCreateBody(serializers.ModelSerializer): + organization = serializers.UUIDField(help_text=IDHelpText) + def to_form_paras(self): custom_paras = to_form_paras(self) @@ -118,21 +120,16 @@ class Meta: model = Agent fields = ( "name", - "capacity", - "node_capacity", - "log_level", "type", - "schedulable", - "ip", - "image", + "urls", "config_file", + "organization", ) extra_kwargs = { - "capacity": {"required": True}, - "node_capacity": {"required": True}, "type": {"required": True}, - "ip": {"required": True}, - "image": {"required": True}, + "urls": {"required": True}, + "name": {"required": True}, + "organization": {"required": False}, "config_file": { "required": False, "validators": [ @@ -145,13 +142,7 @@ class Meta: } def validate(self, attrs): - capacity = attrs.get("capacity") - node_capacity = attrs.get("node_capacity") - if node_capacity < capacity: - raise serializers.ValidationError( - "Node capacity must larger than capacity" - ) - + pass return attrs @@ -187,42 +178,29 @@ class AgentUpdateBody(AgentPatchBody): class AgentResponseSerializer(AgentIDSerializer, serializers.ModelSerializer): - organization_id = serializers.UUIDField( + organization = serializers.UUIDField( help_text="Organization ID", required=False, allow_null=True ) - config_file = serializers.URLField( - help_text="Config file of agent", required=False - ) class Meta: model = Agent fields = ( "id", "name", - "capacity", - "node_capacity", "status", "created_at", - "log_level", "type", - "schedulable", - "organization_id", - "config_file", - "ip", - "image", + "urls", + "organization", ) extra_kwargs = { "id": {"required": True}, "name": {"required": True}, "status": {"required": True}, - "capacity": {"required": True}, - "node_capacity": {"required": True}, "created_at": {"required": True, "read_only": False}, "type": {"required": True}, - "log_level": {"required": True}, - "schedulable": {"required": True}, - "ip": {"required": True}, - "image": {"required": True}, + "organization": {"required": True}, + "urls": {"required": True}, } diff --git a/src/api-engine/api/routes/agent/views.py b/src/api-engine/api/routes/agent/views.py index 6f2739d14..f5fdc98aa 100644 --- a/src/api-engine/api/routes/agent/views.py +++ b/src/api-engine/api/routes/agent/views.py @@ -22,7 +22,7 @@ NoResource, ResourceInUse, ) -from api.models import Agent, KubernetesConfig +from api.models import Agent, KubernetesConfig, Organization from api.routes.agent.serializers import ( AgentQuery, AgentListResponse, @@ -39,6 +39,7 @@ class AgentViewSet(viewsets.ViewSet): + """Class represents agent related operations.""" authentication_classes = (JSONWebTokenAuthentication,) def get_permissions(self): @@ -62,7 +63,9 @@ def list(self, request): """ List Agents - Filter agents with query parameters. + :param request: query parameter + :return: agent list + :rtype: list """ serializer = AgentQuery(data=request.GET) if serializer.is_valid(raise_exception=True): @@ -74,18 +77,18 @@ def list(self, request): organization = serializer.validated_data.get("organization") query_filters = {} - if organization: - if not request.user.is_operator: - raise PermissionDenied() - query_filters.update({"organization": organization}) - else: - org_name = ( - request.user.organization.name - if request.user.organization - else "" - ) - if request.user.is_administrator: - query_filters.update({"organization__name": org_name}) + # if organization: + # if not request.user.is_operator: + # raise PermissionDenied() + # query_filters.update({"organization": organization}) + # else: + # org_name = ( + # request.user.organization.name + # if request.user.organization + # else "" + # ) + # if request.user.is_administrator: + # query_filters.update({"organization__name": org_name}) if name: query_filters.update({"name__icontains": name}) if agent_status: @@ -100,13 +103,6 @@ def list(self, request): agent_list = [] for agent in agents: agent_dict = agent.__dict__ - agent_dict.update( - { - "config_file": request.build_absolute_uri( - agent.config_file.url - ) - } - ) agent_list.append(agent_dict) response = AgentListResponse( @@ -127,45 +123,41 @@ def create(self, request): """ Create Agent - Create new agent + :param request: create parameter + :return: agent ID + :rtype: uuid """ serializer = AgentCreateBody(data=request.data) if serializer.is_valid(raise_exception=True): name = serializer.validated_data.get("name") - capacity = serializer.validated_data.get("capacity") - node_capacity = serializer.validated_data.get("node_capacity") - log_level = serializer.validated_data.get("log_level") agent_type = serializer.validated_data.get("type") - schedulable = serializer.validated_data.get("schedulable") - parameters = serializer.validated_data.get("parameters") - ip = serializer.validated_data.get("ip") - image = serializer.validated_data.get("image") + urls = serializer.validated_data.get("urls") config_file = serializer.validated_data.get("config_file") + organization = serializer.validated_data.get("organization") body = { - "capacity": capacity, - "node_capacity": node_capacity, "type": agent_type, - "ip": ip, - "image": image, + "urls": urls, + "name": name, } if name: agent_count = Agent.objects.filter( - name=name, organization=request.user.organization + name=name ).count() if agent_count > 0: raise ResourceExists( detail="Name %s already exists" % name ) body.update({"name": name}) - if schedulable is not None: - body.update({"schedulable": schedulable}) - if log_level is not None: - body.update({"log_level": log_level}) - if parameters is not None: - body.update({"parameters": parameters}) + if config_file is not None: body.update({"config_file": config_file}) + if organization is not None: + org = Organization.objects.get(id=organization) + if org.organization.all(): + raise ResourceExists + else: + body.update({"organization": org}) agent = Agent(**body) agent.save() @@ -185,7 +177,10 @@ def retrieve(self, request, pk=None): """ Retrieve agent - Retrieve agent + :param request: destory parameter + :param pk: primary key + :return: none + :rtype: rest_framework.status """ try: if request.user.is_operator: From aef41f8f3574df7cbe1983458f8e82ce7efc2a69 Mon Sep 17 00:00:00 2001 From: Baohua Yang Date: Mon, 25 Jan 2021 22:16:34 -0800 Subject: [PATCH 6/8] Clean the potential existing msp and tls paths When the msp or tls path exists, currently, the init.sh will merge the parsed files into it. This may trigger peer's booting failure. This patchset fixes the issue by cleaning the path when the corresponding env variable exists and the target path is not empty. Signed-off-by: Baohua Yang --- build_image/docker/cello-hlf/scripts/init.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build_image/docker/cello-hlf/scripts/init.sh b/build_image/docker/cello-hlf/scripts/init.sh index 09b739e41..aabc3d551 100644 --- a/build_image/docker/cello-hlf/scripts/init.sh +++ b/build_image/docker/cello-hlf/scripts/init.sh @@ -36,6 +36,14 @@ cmd=$1 # The path to store the files cfg_path=${FABRIC_CFG_PATH:-/etc/hyperledger/fabric} +# Clean the potential existing msp and tls paths first +if [ ! -z "${HLF_NODE_MSP}" ] && [ -d ${cfg_path}/msp ]; then + mv ${cfg_path}/msp ${cfg_path}/msp.bak +fi +if [ ! -z "${HLF_NODE_TLS}" ] && [ -d ${cfg_path}/tls ]; then + mv ${cfg_path}/tls ${cfg_path}/tls.bak +fi + # Read each file from env and store under the ${cfg_path} for name in HLF_NODE_MSP \ HLF_NODE_TLS \ From 28402c35a88e19e4b3067ae18d35a2251a5e2ca5 Mon Sep 17 00:00:00 2001 From: Baohua Yang Date: Tue, 26 Jan 2021 23:12:24 -0800 Subject: [PATCH 7/8] Update Maintainers Nominate Qiang Xu as new maintainer. He has leaded the user dashboard development for quite a while, and actively attend the meeting discussions with code contributions. Retire inactive maintainer Luke Chen. Thanks a lot for his contributions. Signed-off-by: Baohua Yang --- MAINTAINERS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index cb19c8501..66d3ab9dd 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -5,6 +5,12 @@ | Baohua Yang | yeasy | baohua | yangbaohua@gmail.com | | Haitao Yue | hightall | hightall | hightallyht@gmail.com | | Tong Li | tongli | tongli | litong01@us.ibm.com | +| Qiang Xu | XuHugo | XuHugo | xq-310@163.com | + +## Retired Maintainers + +| Name | GitHub | RocketChat | Email | +|---|---|---|---| | Luke Chen | LordGoodman | luke_chen | jiahaochen1993@gmail.com | Creative Commons License
This work is licensed under a Creative Commons Attribution 4.0 International License. From 8bed10dd0063c12c7c79ddc79a85f6a5926754d2 Mon Sep 17 00:00:00 2001 From: Baohua Yang Date: Thu, 28 Jan 2021 14:03:05 -0800 Subject: [PATCH 8/8] Fix cello-hlf image * Fix python version to use python3; * Fix the manifest-tool path with latest version; * Other clean ups on comments. Signed-off-by: Baohua Yang --- build_image/docker/cello-hlf/Dockerfile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/build_image/docker/cello-hlf/Dockerfile b/build_image/docker/cello-hlf/Dockerfile index 641bedcde..248a286a9 100644 --- a/build_image/docker/cello-hlf/Dockerfile +++ b/build_image/docker/cello-hlf/Dockerfile @@ -36,7 +36,7 @@ ENV BASE_VERSION=2.2.0 ENV PROJECT_VERSION=2.2.0 ENV HLF_CA_VERSION=1.4.8 -# generic environment (core.yaml) for builder and runtime: builder: $(DOCKER_NS)/fabric-ccenv:$(TWO_DIGIT_VERSION) +# generic environment (core.yaml) for builder and runtime: e.g., builder: $(DOCKER_NS)/fabric-ccenv:$(TWO_DIGIT_VERSION), golang, java, node ENV DOCKER_NS=hyperledger ENV TWO_DIGIT_VERSION=2.2 @@ -80,12 +80,12 @@ RUN mkdir -p /var/hyperledger/production \ # Install development dependencies RUN apt-get update \ - && apt-get install -y apt-utils python-dev \ + && apt-get install -y apt-utils python3-dev \ && apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev libyaml-dev libltdl-dev libtool \ - && apt-get install -y python-pip \ + && apt-get install -y python3-pip \ && apt-get install -y vim tree jq unzip \ - && pip install behave nose docker-compose \ - && pip install pyinotify \ + && pip3 install behave nose docker-compose \ + && pip3 install pyinotify \ && rm -rf /var/cache/apt # Install yq to update config for fabric-ca @@ -99,9 +99,8 @@ RUN go get github.com/golang/protobuf/protoc-gen-go \ && go get github.com/AlekSi/gocov-xml \ && go get golang.org/x/tools/cmd/goimports \ && go get golang.org/x/lint/golint \ - && go get github.com/estesp/manifest-tool \ + && go get github.com/estesp/manifest-tool/... \ && go get github.com/client9/misspell/cmd/misspell \ - && go get github.com/estesp/manifest-tool \ && go get github.com/onsi/ginkgo/ginkgo # Clone the Hyperledger Fabric code and cp sample config files