[ISSUE #4848]Storage adjustment for swagger type API documents: chang…

…e from local cache to database. (#4849)

* apidoc sql

* refact

* commit

* [Task] Shenyu-admin: Fix API document failed to build because of NPE.

* [Task] Shenyu-admin: Fix API document failed to build because of NPE.

* solve conficts,modify LICENSE.

* delete useless code.

* delete useless code.

* commit

* [ISSUE #3843]admin apidoc fix: the required attribute prompt is incorrect when micro service parameter uses "@ApiModelProperty".

* commit

* [shenyu-examples]add swagger to the example project to test the apidoc function of the gateway management system.

* commit

* commit

* commit

* [ISSUE #4690]Supports gzip compression in response to HTTP requests.

* [examples]Add Swagger sample project to demonstrate automatic pull interface documentation.

* delete exapmple

* delete useless code.

* delete useless code.

* swagger apidoc from local cache to database persistence.

* Storage adjustment for swagger type documents :from local cache to database persistence.

* fix get httpMethod

* Fix unit testing.

* Fix unit testing.

* Fix unit testing.

* code style.

* Fix unit testing.

* delete unused imports.

---------

Co-authored-by: lianjunwei <lianjunwei@didiglobal.com>
Co-authored-by: dragon-zhang <zhangzicheng@apache.org>
Co-authored-by: xiaoyu <xiaoyu@apache.org>

shenyu-admin/src/main/java/org/apache/shenyu/admin/controller/ApiDocController.java

@@ -108,7 +108,6 @@ private Function<DocInfo, MenuProject> getMenuAndDocInfo() {
                     List<MenuDocItem> docItems = docModule.getDocItems().stream()
                         .map(docItem -> {
                             MenuDocItem menuDocItem = new MenuDocItem();
-                            menuDocItem.setId(docItem.getId());
                             menuDocItem.setLabel(docItem.getSummary());
                             menuDocItem.setName(docItem.getName());
                             return menuDocItem;

shenyu-admin/src/main/java/org/apache/shenyu/admin/model/bean/DocInfo.java

@@ -28,6 +28,8 @@ public class DocInfo {
 
     private String clusterName;
 
+    private String contextPath;
+
     private List<DocModule> docModuleList;
 
     /**
@@ -66,6 +68,24 @@ public void setClusterName(final String clusterName) {
         this.clusterName = clusterName;
     }
 
+    /**
+     * get contextPath.
+     *
+     * @return contextPath
+     */
+    public String getContextPath() {
+        return contextPath;
+    }
+
+    /**
+     * set contextPath.
+     *
+     * @param contextPath contextPath
+     */
+    public void setContextPath(final String contextPath) {
+        this.contextPath = contextPath;
+    }
+
     /**
      * getDocModuleList.
      *

shenyu-admin/src/main/java/org/apache/shenyu/admin/model/bean/DocItem.java

@@ -27,8 +27,6 @@
  */
 public class DocItem {
 
-    private String id;
-
     private String module;
 
     private String name;
@@ -47,6 +45,8 @@ public class DocItem {
      */
     private Collection<String> httpMethodList;
 
+    private Collection<String> consumes;
+
     private Collection<String> produces;
 
     /**
@@ -81,24 +81,6 @@ public boolean isUploadRequest() {
         return multiple || upload;
     }
 
-    /**
-     * getId.
-     *
-     * @return String
-     */
-    public String getId() {
-        return id;
-    }
-
-    /**
-     * setId.
-     *
-     * @param id id
-     */
-    public void setId(final String id) {
-        this.id = id;
-    }
-
     /**
      * getModule.
      *
@@ -207,6 +189,24 @@ public void setHttpMethodList(final Collection<String> httpMethodList) {
         this.httpMethodList = httpMethodList;
     }
 
+    /**
+     * get consumes.
+     *
+     * @return consumes
+     */
+    public Collection<String> getConsumes() {
+        return consumes;
+    }
+
+    /**
+     * set consumes.
+     *
+     * @param consumes consumes
+     */
+    public void setConsumes(final Collection<String> consumes) {
+        this.consumes = consumes;
+    }
+
     /**
      * getProduces.
      *

shenyu-admin/src/main/java/org/apache/shenyu/admin/service/manager/DocManager.java

@@ -36,14 +36,6 @@ public interface DocManager {
      */
     void addDocInfo(String serviceId, String docJson, Consumer<DocInfo> callback);
 
-    /**
-     * get docInfo by title.
-     *
-     * @param title title
-     * @return DocInfo
-     */
-    DocInfo getByTitle(String title);
-
     /**
      * getDocItem.
      *

shenyu-admin/src/main/java/org/apache/shenyu/admin/service/manager/ServiceDocManager.java → shenyu-admin/src/main/java/org/apache/shenyu/admin/service/manager/PullSwaggerDocService.java

@@ -21,9 +21,9 @@
 import org.apache.shenyu.admin.model.bean.UpstreamInstance;
 
 /**
- * Service document Manager.
+ * pull service api document Manager.
  */
-public interface ServiceDocManager {
+public interface PullSwaggerDocService {
 
     /**
      * pull API document.

shenyu-admin/src/main/java/org/apache/shenyu/admin/service/manager/RegisterApiDocService.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shenyu.admin.service.manager;
+
+import org.apache.shenyu.register.common.dto.ApiDocRegisterDTO;
+
+/**
+ * register api document manager.
+ */
+public interface RegisterApiDocService {
+
+    /**
+     * register api document.
+     * @param apiDocRegisterDTO apiDocRegisterDTO
+     */
+    void registerApiDocument(ApiDocRegisterDTO apiDocRegisterDTO);
+}

shenyu-admin/src/main/java/org/apache/shenyu/admin/service/manager/impl/DocManagerImpl.java

@@ -19,22 +19,33 @@
 
 import java.nio.charset.StandardCharsets;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Consumer;
 
 import com.google.gson.JsonObject;
+import javax.annotation.Resource;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.shenyu.admin.model.bean.DocInfo;
 import org.apache.shenyu.admin.model.bean.DocItem;
 import org.apache.shenyu.admin.model.bean.DocModule;
 import org.apache.shenyu.admin.service.manager.DocManager;
 import org.apache.shenyu.admin.service.manager.DocParser;
+import org.apache.shenyu.admin.service.manager.RegisterApiDocService;
+import org.apache.shenyu.common.enums.ApiHttpMethodEnum;
+import org.apache.shenyu.common.enums.ApiSourceEnum;
+import org.apache.shenyu.common.enums.ApiStateEnum;
+import org.apache.shenyu.common.enums.RpcTypeEnum;
 import org.apache.shenyu.common.utils.GsonUtils;
+import org.apache.shenyu.common.utils.JsonUtils;
+import org.apache.shenyu.register.common.dto.ApiDocRegisterDTO;
+import org.apache.shenyu.register.common.enums.EventType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
@@ -47,6 +58,13 @@
 public class DocManagerImpl implements DocManager {
     private static final Logger LOG = LoggerFactory.getLogger(DocManagerImpl.class);
 
+    private static final String API_DOC_VERSION = "v0.01";
+
+    /**
+     * The constant HTTP.
+     */
+    private static final String HTTP = "http://";
+
     /**
      * key:title, value:docInfo.
      */
@@ -64,6 +82,9 @@ public class DocManagerImpl implements DocManager {
 
     private static final DocParser SWAGGER_DOC_PARSER = new SwaggerDocParser();
 
+    @Resource
+    private RegisterApiDocService registerApiDocService;
+
     /**
      * add docInfo.
      *
@@ -81,39 +102,85 @@ public void addDocInfo(final String clusterName, final String docInfoJson, final
         if (Objects.equals(newMd5, oldMd5)) {
             return;
         }
-        CLUSTER_MD5_MAP.put(clusterName, newMd5);
+//        CLUSTER_MD5_MAP.put(clusterName, newMd5);
         DocInfo docInfo = getDocInfo(clusterName, docInfoJson);
         if (Objects.isNull(docInfo) || CollectionUtils.isEmpty(docInfo.getDocModuleList())) {
             return;
         }
         List<DocModule> docModules = docInfo.getDocModuleList();
         DOC_DEFINITION_MAP.put(docInfo.getTitle(), docInfo);
-        docModules.forEach(docModule -> docModule.getDocItems().forEach(docItem -> ITEM_DOC_MAP.put(docItem.getId(), docItem)));
+        docModules.forEach(docModule -> docModule.getDocItems().forEach(docItem -> {
+            ApiDocRegisterDTO build = ApiDocRegisterDTO.builder()
+                .consume(this.getProduceConsume(docItem.getConsumes()))
+                .produce(this.getProduceConsume(docItem.getProduces()))
+                .httpMethod(this.getHttpMethod(docItem))
+                .contextPath(docInfo.getContextPath())
+                .ext(this.buildExtJson(docInfo, docItem))
+                .document(JsonUtils.toJson(docItem))
+                .rpcType(RpcTypeEnum.HTTP.getName())
+                .version(API_DOC_VERSION)
+                .apiDesc(docItem.getDescription())
+                .tags(Collections.singletonList(docInfo.getContextPath()))
+                .apiPath(docItem.getName())
+                .apiSource(ApiSourceEnum.SWAGGER.getValue())
+                .state(ApiStateEnum.PUBLISHED.getState())
+                .apiOwner("admin")
+                .eventType(EventType.REGISTER)
+                .build();
+
+            registerApiDocService.registerApiDocument(build);
+        }));
+
         callback.accept(docInfo);
     }
 
+    private String getProduceConsume(final Collection<String> list) {
+        String res = StringUtils.EMPTY;
+        if (Objects.nonNull(list)) {
+            Optional<String> first = list.stream().findFirst();
+            if (first.isPresent()) {
+                res = first.get();
+            }
+        }
+        return StringUtils.isNotEmpty(res) ? res : "*/*";
+    }
+
+    private Integer getHttpMethod(final DocItem docItem) {
+        Integer httpMethod = null;
+        Optional<String> first = docItem.getHttpMethodList().stream().findFirst();
+        if (first.isPresent()) {
+            String method = docItem.getHttpMethodList().size() == 1 ? StringUtils.upperCase(first.get()) : ApiHttpMethodEnum.GET.getName();
+            httpMethod = ApiHttpMethodEnum.getValueByName(method);
+        }
+        return httpMethod;
+    }
+
     private DocInfo getDocInfo(final String clusterName, final String docInfoJson) {
         try {
             JsonObject docRoot = GsonUtils.getInstance().fromJson(docInfoJson, JsonObject.class);
-            docRoot.addProperty("basePath", "/" + clusterName);
+            String contexPath = "/" + clusterName;
+            docRoot.addProperty("basePath", contexPath);
             DocInfo docInfo = SWAGGER_DOC_PARSER.parseJson(docRoot);
             docInfo.setClusterName(clusterName);
+            docInfo.setContextPath(contexPath);
             return docInfo;
         } catch (Exception e) {
             LOG.error("getDocInfo error= ", e);
             return null;
         }
     }
 
-    /**
-     * get doc By Title.
-     *
-     * @param title title
-     * @return DocInfo
-     */
-    @Override
-    public DocInfo getByTitle(final String title) {
-        return DOC_DEFINITION_MAP.get(title);
+    private String buildExtJson(final DocInfo docInfo, final DocItem docItem) {
+        ApiDocRegisterDTO.ApiExt ext = new ApiDocRegisterDTO.ApiExt();
+        ext.setHost("host");
+        ext.setPort(null);
+        ext.setServiceName(docInfo.getClusterName());
+        ext.setMethodName(docItem.getName());
+        ext.setParameterTypes("");
+        ext.setRpcExt(null);
+        ext.setAddPrefixed(false);
+        ext.setProtocol(HTTP);
+        return GsonUtils.getInstance().toJson(ext);
     }
 
     /**

shenyu-admin/src/main/java/org/apache/shenyu/admin/service/manager/impl/LoadServiceDocEntryImpl.java

@@ -41,7 +41,7 @@
 import org.apache.shenyu.admin.service.ShenyuDictService;
 import org.apache.shenyu.admin.service.converter.SelectorHandleConverterFactor;
 import org.apache.shenyu.admin.service.manager.LoadServiceDocEntry;
-import org.apache.shenyu.admin.service.manager.ServiceDocManager;
+import org.apache.shenyu.admin.service.manager.PullSwaggerDocService;
 import org.apache.shenyu.common.constant.AdminConstants;
 import org.apache.shenyu.common.dto.SelectorData;
 import org.apache.shenyu.common.dto.convert.selector.CommonUpstream;
@@ -69,19 +69,19 @@ public class LoadServiceDocEntryImpl implements LoadServiceDocEntry {
 
     private final PluginMapper pluginMapper;
 
-    private final ServiceDocManager serviceDocManager;
+    private final PullSwaggerDocService pullSwaggerDocService;
 
     private final ShenyuDictService shenyuDictService;
 
     public LoadServiceDocEntryImpl(final SelectorService selectorService,
                                    final SelectorHandleConverterFactor converterFactor,
                                    final PluginMapper pluginMapper,
-                                   final ServiceDocManager serviceDocManager,
+                                   final PullSwaggerDocService pullSwaggerDocService,
                                    final ShenyuDictService shenyuDictService) {
         this.selectorService = selectorService;
         this.converterFactor = converterFactor;
         this.pluginMapper = pluginMapper;
-        this.serviceDocManager = serviceDocManager;
+        this.pullSwaggerDocService = pullSwaggerDocService;
         this.shenyuDictService = shenyuDictService;
     }
 
@@ -97,7 +97,7 @@ public synchronized void loadApiDocument() {
         }
         final Set<UpstreamInstance> currentServices = new HashSet<>(serviceList);
         LOG.info("load api document  serviceList={}", JsonUtils.toJson(currentServices));
-        serviceDocManager.pullApiDocument(currentServices);
+        pullSwaggerDocService.pullApiDocument(currentServices);
     }
 
     @Override
@@ -113,7 +113,7 @@ public void loadDocOnSelectorChanged(final List<SelectorData> changedList, final
             }
             final Set<UpstreamInstance> currentServices = new HashSet<>(serviceList);
             LOG.info("loadDocOnSelectorChanged serviceList={}", JsonUtils.toJson(currentServices));
-            serviceDocManager.pullApiDocument(currentServices);
+            pullSwaggerDocService.pullApiDocument(currentServices);
         }
     }