feat: Add apollo audit log common solution backend (#4985)

* 7.3

* plc success except jpa part

* successfully run by EntityScan annotation(not a good way)

* successfully run by add packages to bean registrar

* 'app create' audit log and query poc success

* App create/update/delete poc success.

* add License

* add License

* try to pass tests

* try to pass tests

* fix ${revision} to ${project.version}

* add no-op implement and no-op autoconfiguration.
add ApolloAuditLogApi for manually logging needs.
add ApolloAuditLogDataInfluenceTableId (maybe multi-primary-key in the future).
add ApolloAuditController by @bean registered

* rebase and add license

* add audit controller

* - change controller
- add @DomainEvents to BaseEntity.java
- change the way that data influences append
- add annotations and codes to make app audited
- change some classes' name
- make api to query and record api
- add event and listener to catch data influences

* - make ApolloAuditHttpInterceptor uses Api while not tracer
- try to reduce the affection of audit module on other modules

* - retry workflows

* - change for code review

* - basically finish front-end part.

* - fix for hound

* - update

* - update

* - add ApolloAuditLogApiNoOpImpl.class to @ContextConfiguration at AppServiceTest.class

* - add license and fix for hound

* - fix front-end bug
- add index for 2 new table

* - fix front-end bug

* - remove query for auditing AppNamespace

* fix for changed requested

* - change: always record if the method have no annotation
- fix: little few bugs

* - add test cover manually add dataInfluence when delete app
- delete filter which is no longer needed

* - try to write part of tests in impl
- add Exception

* - finish ApolloAuditLogApiJpaImplTest
- add license

* - finish most part of context's unit-tests

* - finish some unit-tests

* - fix database design bug

- remove exception which would cause unexpected transaction-rollback

* - add prompt sheet for disabling audit log feature

- add API for get isEnabled of audit log feature

- fix sql problem and add it to delta

- remove 2 classes

* - fix test bugs

* - change the way of getting op-type for data influences

* - enhance front-end design

* - enhance front-end design

* - add AuditLog Search dropdown

* - add any-match data influences for batch-delete operation auditing

* - fix bugs

* - fix front-end bugs
- add javadoc
- Update ApolloAuditController to return ApolloAuditProperties directly

* - fix test failure

* - fix test failure

* - update javadoc
- add README.md

* - update README

* - update README

apollo-adminservice/pom.xml

@@ -35,6 +35,10 @@
 			<groupId>com.ctrip.framework.apollo</groupId>
 			<artifactId>apollo-biz</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>com.ctrip.framework.apollo</groupId>
+			<artifactId>apollo-audit-spring-boot-starter</artifactId>
+		</dependency>
 		<!-- end of apollo -->
 		<dependency>
 			<groupId>org.springframework.cloud</groupId>

apollo-adminservice/src/main/resources/application.properties

@@ -22,3 +22,7 @@
 
 # You may change the following config to activate different database profiles like h2/postgres
 spring.profiles.group.github = mysql
+
+# true: enabled the new feature of audit log
+# false/missing: disable it
+apollo.audit.log.enabled = true

apollo-assembly/src/main/java/com/ctrip/framework/apollo/assembly/ApolloApplication.java

@@ -17,6 +17,7 @@
 package com.ctrip.framework.apollo.assembly;
 
 import com.ctrip.framework.apollo.adminservice.AdminServiceApplication;
+import com.ctrip.framework.apollo.audit.configuration.ApolloAuditAutoConfiguration;
 import com.ctrip.framework.apollo.configservice.ConfigServiceApplication;
 import com.ctrip.framework.apollo.portal.PortalApplication;
 
@@ -31,7 +32,7 @@
 import org.springframework.context.ConfigurableApplicationContext;
 
 @SpringBootApplication(exclude = {DataSourceAutoConfiguration.class,
-    HibernateJpaAutoConfiguration.class})
+    HibernateJpaAutoConfiguration.class, ApolloAuditAutoConfiguration.class})
 public class ApolloApplication {
 
   private static final Logger logger = LoggerFactory.getLogger(ApolloApplication.class);

apollo-assembly/src/main/resources/application.properties

@@ -22,3 +22,7 @@
 
 # You may change the following config to activate different database profiles like h2/postgres
 spring.profiles.group.github = mysql
+
+# true: enabled the new feature of audit log
+# false/missing: disable it
+apollo.audit.log.enabled = true

apollo-audit/README.md

@@ -0,0 +1,147 @@
+# Features: Apollo-Audit-Log
+
+This module provides audit log functions for other Apollo modules.
+
+Only apolloconfig's developer need to read it, 
+
+apolloconfig's user doesn't need.
+
+## How to enable/disable
+
+We can switch this module freely by properties:
+
+by adding properties to application.properties:
+
+```
+# true: enabled the new feature of audit log
+# false/missing: disable it
+apollo.audit.log.enabled = true
+```
+
+## How to generate audit log
+
+### Append an AuditLog
+
+Through an AuditLog, we have the ability to record **Who, When, Why, Where, What and How** operates something.
+
+We can do this by using annotations:
+
+```java
+@ApolloAuditLog(type=OpType.CREATE,name="App.create")
+public App create() {
+  // ...
+}
+```
+
+Through this, an AuditLog will be created and its AuditScope will be activated during the execution of this method.
+
+Equally, we can use ApolloAuditLogApi to do this manually:
+
+```java
+public App create() {
+  try(AutoCloseable auditScope = api.appendAuditLog(type, name)) {
+    // ...
+  }
+}
+/**************OR**************/
+public App create() {
+  Autocloseable auditScope = api.appendAuditLog(type, name);
+  // ...
+  auditScope.close();
+}
+```
+
+The only thing you need to pay attention to is that you need to close this scope manually～
+
+### Append DataInfluence
+
+This function can also be implemented automatically and manually.
+
+There is a corresponding relationship between DataInfluences and a certain AuditLog, and they are caused by this AuditLog. But not all AuditLogs will generate DataInfluences!
+
+#### Mark which data change
+
+First, we need to add audit-bean-definition to class of the entity you want to audit:
+
+```java
+@ApolloAuditLogDataInfluenceTable(tableName = "App")
+public class App extends BaseEntity {
+  @ApolloAuditLogDataInfluenceTableField(fieldName = "Name")
+  private String name;
+  private String orgId;
+}
+```
+
+In class App, we define that its data-influence table' name is "App", the field "name" needs to be audited and its audit field name in the table "App" is "Name". The field "orgId" is no need to be audited.
+
+Second, use API's method to append it:
+
+Actually we don't need to manually call it. We can depend on the DomainEvents that pre-set in BaseEntity:
+
+```java
+@DomainEvents
+public Collection<ApolloAuditLogDataInfluenceEvent> domainEvents() {
+  return Collections.singletonList(new ApolloAuditLogDataInfluenceEvent(this.getClass(), this));
+}
+```
+
+And this will call appendDataInfluences automatically by the listener.
+
+#### Manually
+
+```java
+/**
+ * Append DataInfluences by a list of entities needs to be audited, and their
+ * audit-bean-definition.
+ */
+ApolloAuditLogApi.appendDataInfluences(List<Object> entities, Class<?> beanDefinition);
+```
+
+Just call the api method in an active scope, the data influences will combine with the log automatically.
+
+```java
+public App create() {
+  try(AutoCloseable auditScope = api.appendAuditLog(type, name)) {
+    // ...
+    api.appendDataInfluences(appList, App.class);
+  	// or.
+  	api.appendDataInfluence("App","10001","Name","xxx");
+  }
+}
+```
+
+#### some tricky situations
+
+Yet, sometimes we can't catch the domain events like some operations that directly change database fields. We can use annotations to catch the input parameters:
+
+```java
+@ApolloAuditLog(type=OpType.DELETE,name="AppNamespace.batchDeleteByAppId")
+public AppNamespace batchDeleteByAppId(
+  @ApolloAuditLogDataInfluence
+  @ApolloAuditLogDataInfluenceTable(tableName="AppNamespace")
+  @ApolloAuditLogDataInfluenceTableField(fieldName="AppId") String appId) {
+  // ...
+}
+```
+
+This will generate a special data influence. It means that all entities matching the input parameter value have been affected.
+
+## How to verify the audit-log work
+
+### check-by-UI
+
+The entrance of audit log UI is in Admin Tools.
+
+Then, we can check if the AuditLogs are created properly by searching or just find in table below.
+
+Then, check in the trace detail page.
+
+We can check if the relationship between AuditLogs are correct and the DataInfluences caused by certain AuditLog is logically established.
+
+In the rightmost column, we can view the historical operation records of the specified field's value. Null means being deleted~
+
+### check-by-database
+
+The databases are in ApolloPortalDB, the table `AuditLog` and `AuditLogDataInfluence`.
+
+We can verify if the parent/followsfrom relationships are in line with our expectations.

apollo-audit/apollo-audit-annotation/pom.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2023 Apollo Authors
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>apollo-audit</artifactId>
+    <groupId>com.ctrip.framework.apollo</groupId>
+    <version>${revision}</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>apollo-audit-annotation</artifactId>
+  <version>${revision}</version>
+
+</project>

apollo-audit/apollo-audit-annotation/src/main/java/com/ctrip/framework/apollo/audit/annotation/ApolloAuditLog.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2023 Apollo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.ctrip.framework.apollo.audit.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Mark which method should be audited, add to controller or service's method.
+ * <p></p>
+ * Define the attributes of the operation for persisting and querying. When adding to controller's
+ * methods, suggested that don't set name, and it will automatically be set to request's url.
+ * <p></p>
+ * Example usage:
+ * <pre>
+ * {@code
+ * @ApolloAuditLog(type=OpType.CREATE,name="App.create")
+ * public App create() {
+ *   // ...
+ * }
+ * }
+ * </pre>
+ *
+ * @author luke0125
+ * @since 2.2.0
+ */
+@Target(ElementType.METHOD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ApolloAuditLog {
+
+  /**
+   * Define the type of operation.
+   *
+   * @return operation type
+   */
+  OpType type();
+
+  /**
+   * Define the name of operation. The requested URL will be taken by default if no specific name is
+   * specified.
+   *
+   * @return operation name
+   */
+  String name() default "";
+
+  /**
+   * Define the description of operation. Default is "no description".
+   *
+   * @return operation description
+   */
+  String description() default "no description";
+}
+

apollo-audit/apollo-audit-annotation/src/main/java/com/ctrip/framework/apollo/audit/annotation/ApolloAuditLogDataInfluence.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2023 Apollo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.ctrip.framework.apollo.audit.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Combine with {@link ApolloAuditLog}, mark which method's parameter is audit log's data change.
+ * <p></p>
+ * Example usage:
+ * <pre>
+ * {@code
+ * @ApolloAuditLog(type=OpType.DELETE,name="AppNamespace.batchDeleteByAppId")
+ * public AppNamespace batchDeleteByAppId(
+ *            @ApolloAuditLogDataInfluence String appId) {
+ *   // ...
+ * }
+ * }
+ * </pre>
+ *
+ * @author luke0125
+ * @since 2.2.0
+ */
+@Target(ElementType.PARAMETER)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ApolloAuditLogDataInfluence {
+
+}