diff --git a/MoquiConf.xml b/MoquiConf.xml index c2ce961..e026cb4 100644 --- a/MoquiConf.xml +++ b/MoquiConf.xml @@ -2,4 +2,15 @@ + + + + + + + + /rest/s1/shopify/webhook/* + + + \ No newline at end of file diff --git a/README.md b/README.md index 8e1bd5b..15df879 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Set of services, templates and configuration to integrate with Shopify Bulk Expo This integration enables you, 1. To configure and poll shopify jsonl feeds from SFTP, stage and upload on shopify and run the intended shopify bulk mutation operation. 2. To configure and send shopify bulk queries. + It also polls the running bulk operation status and download and stores the result file locally. Support for BULK_OPERATION_FINISH webhook will be implemented in next phase. @@ -167,7 +168,9 @@ Supported bulk mutations and configuration, sendServiceName="co.hotwax.shopify.system.ShopifySystemMessageServices.send#BulkMutationSystemMessage" sendPath="component://shopify-connector/template/graphQL/BulkUpdateProductTags.ftl" consumeServiceName="co.hotwax.shopify.system.ShopifySystemMessageServices.consume#BulkOperationResult" - receivePath="${contentRoot}/hotwax/shopify/ProductTagsFeed/result/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"/> + receivePath="${contentRoot}/hotwax/shopify/ProductTagsFeed/result/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + receivePath="${contentRoot}/hotwax/shopify/ProductVariantsFeed/result/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + - @@ -263,17 +268,17 @@ You could configure following default parameters and any additional parameters a ```aidl - - - - ``` @@ -286,7 +291,9 @@ You could configure following default parameters and any additional parameters a sendServiceName="co.hotwax.shopify.system.ShopifySystemMessageServices.send#BulkQuerySystemMessage" sendPath="component://shopify-connector/template/graphQL/BulkVariantsMetafieldQuery.ftl" consumeServiceName="co.hotwax.shopify.system.ShopifySystemMessageServices.consume#BulkOperationResult" - receivePath="${contentRoot}/hotwax/shopify/BulkVariantsMetafieldFeed/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"/> + receivePath="${contentRoot}/hotwax/shopify/BulkVariantsMetafieldFeed/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + receivePath="${contentRoot}/hotwax/shopify/BulkOrderMetafieldsFeed/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + +``` + +## Shopify Webhook Integration + +Set of services and configuration to integrate with Shopify Webhook GraphQL API. +This integration enables you to configure a shopify webhook topic, subscribe/unsubscribe to it, receive payload from the subscribed webhook topic and consume the payload to further process it. + +### Webhook Filter + +**co.hotwax.shopify.ShopifyWebhookFilter**: A filter to verify HMAC for all incoming webhook payloads and set the required attributes on HTTP request upon successful verification. + +#### Configuration + +Folliowing configuration is added to MoquiConf.xml, + +```aidl + + + + + + + /rest/s1/shopify/webhook/* + + + +``` +### Core Services + +1. **create#WebhookSubscription**: Subscribe to shopify webhook topic with your apps callbackUrl (end point). +2. **get#WebhookSubscriptions**: Get a list of all subscribed webhooks filtered by query parameters. +3. **delete#WebhookSubscription**: Unsubscribe a specific webhook topic. +4. **verify#Hmac**: Verify hmac for the received webhook payload. +5. **receive#WebhookPayload**: Receive webhook payload in an incoming SystemMessage of the webhook topics SystemMessageType. +6. **produce#WebhookSubscriptionSystemMessage**: Service to initiate webhook subscription of a specific type by creating a system message. +7. **send#WebhookSubscriptionSystemMessage**: Send service to invoke Create Webhook Subscription API for the System Message. +8. **produce#WebhookSubscriptionDeleteSystemMessage**: Service to initiate delete webhook subscription of a specific type by creating a system message. +9. **send#WebhookSubscriptionDeleteSystemMessage**: Send service to invoke Delete Webhook Subscription API for the System Message. This service first get the webhookSubscriptionId for specified webhook topic and registered callbackUrl and the invokes Delete Webhook Subscription API for the webhookSubscriptionId. + +### Subscribing a Webhook Topic + +1. Following is some global configuration data for webhook subscriptions, + ```aidl + + + + + + ``` +2. Implement a consume service as needed to process webhook payload. In absence of a conusme service, the payload would just be saved as is in an incoming SystemMessage. +3. To subscribe a webhook you need to define following configuration data, + ```aidl + + + + + + + + (https://shopify.dev/docs/api/admin-rest/2023-10/resources/webhook#event-topics) + ``` +4. To subscribe to the webhook invoke _produce#WebhookSubscriptionSystemMessage_ service. + +### Unsubscribing a Webhook Topic + +1. Following is the configuration data for deleting any webhook subscription, + ```aidl + + + + + ``` +2. To unsubscribe webhook invoke _produce#WebhookSubscriptionDeleteSystemMessage_ service. + +### Supported Shopify Webhooks + +#### Bulk Operations Finish (bulk_operations/finish) + +```aidl + + + + + + + ``` \ No newline at end of file diff --git a/data/ShopifyConfigDemoData.xml b/data/ShopifyConfigDemoData.xml index 361b27e..944923d 100644 --- a/data/ShopifyConfigDemoData.xml +++ b/data/ShopifyConfigDemoData.xml @@ -9,8 +9,8 @@ + sendUrl="https://${shopifyHost}/admin/api/${shopifyApiVersion}" sharedSecret="" + accessScopeEnumId="" sendSharedSecret=""/> + + + + + + + + + + + + + \ No newline at end of file diff --git a/data/ShopifySetupSeedData.xml b/data/ShopifySetupSeedData.xml index 24237ff..d43770f 100644 --- a/data/ShopifySetupSeedData.xml +++ b/data/ShopifySetupSeedData.xml @@ -92,7 +92,9 @@ under the License. sendServiceName="co.hotwax.shopify.system.ShopifySystemMessageServices.send#BulkMutationSystemMessage" sendPath="component://shopify-connector/template/graphQL/BulkUpdateProductTags.ftl" consumeServiceName="co.hotwax.shopify.system.ShopifySystemMessageServices.consume#BulkOperationResult" - receivePath="${contentRoot}/shopify/ProductTagsFeed/result/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"/> + receivePath="${contentRoot}/shopify/ProductTagsFeed/result/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + receivePath="${contentRoot}/shopify/ProductVariantsFeed/result/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + receivePath="${contentRoot}/shopify/BulkVariantsMetafieldFeed/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + receivePath="${contentRoot}/shopify/BulkOrderMetafieldsFeed/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data/UpgradeData_v1.2.0.xml b/data/UpgradeData_v1.2.0.xml index 3a4328d..37146be 100644 --- a/data/UpgradeData_v1.2.0.xml +++ b/data/UpgradeData_v1.2.0.xml @@ -56,7 +56,9 @@ sendServiceName="co.hotwax.shopify.system.ShopifySystemMessageServices.send#BulkMutationSystemMessage" sendPath="component://shopify-connector/template/graphQL/BulkUpdateProductTags.ftl" consumeServiceName="co.hotwax.shopify.system.ShopifySystemMessageServices.consume#BulkOperationResult" - receivePath="${contentRoot}/shopify/ProductTagsFeed/result/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"/> + receivePath="${contentRoot}/shopify/ProductTagsFeed/result/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + receivePath="${contentRoot}/shopify/ProductVariantsFeed/result/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + receivePath="${contentRoot}/shopify/BulkVariantsMetafieldFeed/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + receivePath="${contentRoot}/shopify/BulkOrderMetafieldsFeed/BulkOperationResult-${systemMessageId}-${remoteMessageId}-${nowDate}.jsonl"> + + + + + + + + + + + + + + + + + + + + + diff --git a/entity/ShopifyEntities.xml b/entity/ShopifyEntities.xml index d9aa26a..9ed57d4 100644 --- a/entity/ShopifyEntities.xml +++ b/entity/ShopifyEntities.xml @@ -18,7 +18,7 @@ under the License. - + Entity to define any additional parameters w.r.t. SystemMessageType required to process a SystemMessage. Optionally configure systemMessageRemoteId. @@ -27,12 +27,8 @@ under the License. - - - - - - + + @@ -53,6 +49,7 @@ under the License. + diff --git a/service/co/hotwax/shopify/common/ShopifyHelperServices.xml b/service/co/hotwax/shopify/common/ShopifyHelperServices.xml index 4ae99cb..35f9162 100644 --- a/service/co/hotwax/shopify/common/ShopifyHelperServices.xml +++ b/service/co/hotwax/shopify/common/ShopifyHelperServices.xml @@ -84,8 +84,6 @@ under the License. restClient.basicAuth(systemMessageRemote.username, systemMessageRemote.password); } else if (systemMessageRemote.sharedSecret) { restClient.addHeader(systemMessageRemote.authHeaderName?:'X-Shopify-Access-Token',systemMessageRemote.sharedSecret); - } else if (systemMessageRemote.sendSharedSecret) { - restClient.addHeader(systemMessageRemote.authHeaderName?:'X-Shopify-Access-Token',systemMessageRemote.sendSharedSecret); } else { restClient.addHeader(systemMessageRemote.authHeaderName?:'X-Shopify-Access-Token',systemMessageRemote.password); } @@ -111,10 +109,16 @@ under the License. statusCode = restResponse.getStatusCode() headers = restResponse.headers() - ec.logger.info("Shopify X-Shopify-Shop-Api-Call-Limit " + restResponse.headerFirst("X-Shopify-Shop-Api-Call-Limit")); + if (headers.get("X-Shopify-Shop-Api-Call-Limit") != null) { + ec.logger.info("Shopify X-Shopify-Shop-Api-Call-Limit " + restResponse.headerFirst("X-Shopify-Shop-Api-Call-Limit")); + } else { + ec.logger.info("Shopify X-Shopify-Shop-Api-Call-Limit " + restResponse.headerFirst("x-shopify-shop-api-call-limit")); + } //TODO:Handle retry after https://shopify.dev/api/usage/rate-limits#rate-limiting-methods if (headers.get("Retry-After") != null) { ec.logger.warn("Shopify Retry-After " + restResponse.headerFirst("Retry-After")); + } else { + ec.logger.warn("Shopify Retry-After " + restResponse.headerFirst("retry-after")); } if (restResponse.statusCode < 200 || restResponse.statusCode >= 300) { @@ -127,19 +131,24 @@ under the License. ]]> - - - - - - - - + + + + + + + + + + + + + - - + + \ No newline at end of file diff --git a/service/co/hotwax/shopify/system/ShopifySystemMessageServices.xml b/service/co/hotwax/shopify/system/ShopifySystemMessageServices.xml index 316266a..64129b6 100644 --- a/service/co/hotwax/shopify/system/ShopifySystemMessageServices.xml +++ b/service/co/hotwax/shopify/system/ShopifySystemMessageServices.xml @@ -124,7 +124,7 @@ under the License. - + @@ -183,7 +183,6 @@ under the License. - @@ -210,6 +209,15 @@ under the License. + + + + + + + + + @@ -252,7 +260,6 @@ under the License. - @@ -263,7 +270,7 @@ under the License. + in-map="[systemMessageId:systemMessages[0].systemMessageId]" out-map="context"/> @@ -298,7 +305,7 @@ under the License. - + @@ -349,4 +356,26 @@ under the License. + + Consume service to process bulk_operations/finish wehbook payload + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/service/co/hotwax/shopify/webhook/ShopifyWebhookServices.xml b/service/co/hotwax/shopify/webhook/ShopifyWebhookServices.xml new file mode 100644 index 0000000..4aa1d31 --- /dev/null +++ b/service/co/hotwax/shopify/webhook/ShopifyWebhookServices.xml @@ -0,0 +1,295 @@ + + + + + + + Subscribe to shopify webhook topic with a callbackUrl (end point). + + + + + + + + + + + + + + + + + + + + + + + Get a list of all subscribed webhooks filtered by query parameters. + + + + + + + + + + + + + + + + + + + + + + Unsubscribe a specific webhook topic. + + + + + + + + + + + + + + + + + + + + + + + + + + Verify hmac for the received webhook payload. + + + + + + + + + + + + + + + + + + Receive webhook payload in an incoming SystemMessage of the webhook topics SystemMessageType. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Service to initiate webhook subscription of a specific type by creating a system message. + + + + + + + + + + + + + + + + + + + + + + + + + Send service to invoke Create Webhook Subscription API for the System Message. + + + + + + + + + + + + + + + + + + Service to initiate delete webhook subscription of a specific type by creating a system message. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Send service to invoke Delete Webhook Subscription API for the System Message. + This service first get the webhookSubscriptionId for specified webhook topic and registered callbackUrl and the invokes Delete Webhook Subscription API for the webhookSubscriptionId. + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/service/shopify.rest.xml b/service/shopify.rest.xml new file mode 100644 index 0000000..da85dc2 --- /dev/null +++ b/service/shopify.rest.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/groovy/co/hotwax/shopify/ShopifyWebhookFilter.groovy b/src/main/groovy/co/hotwax/shopify/ShopifyWebhookFilter.groovy new file mode 100644 index 0000000..fdfe2f5 --- /dev/null +++ b/src/main/groovy/co/hotwax/shopify/ShopifyWebhookFilter.groovy @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.hotwax.shopify + +import groovy.transform.CompileStatic +import org.moqui.entity.EntityCondition +import org.moqui.entity.EntityList +import org.moqui.entity.EntityValue +import org.moqui.impl.context.ContextJavaUtil +import org.moqui.impl.context.ExecutionContextFactoryImpl +import org.moqui.impl.context.ExecutionContextImpl +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.apache.commons.io.IOUtils + +import javax.servlet.* +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +@CompileStatic +class ShopifyWebhookFilter implements Filter { + + protected static final Logger logger = LoggerFactory.getLogger(ShopifyWebhookFilter.class) + protected FilterConfig filterConfig = null + + ShopifyWebhookFilter() { super() } + + @Override + void init(FilterConfig filterConfig) { + this.filterConfig = filterConfig + } + + @Override + void doFilter(ServletRequest req, ServletResponse resp, FilterChain chain) { + if (!(req instanceof HttpServletRequest) || !(resp instanceof HttpServletResponse)) { + chain.doFilter(req, resp); return + } + + HttpServletRequest request = (HttpServletRequest) req + HttpServletResponse response = (HttpServletResponse) resp + + ServletContext servletContext = req.getServletContext() + + ExecutionContextFactoryImpl ecfi = (ExecutionContextFactoryImpl) servletContext.getAttribute("executionContextFactory") + // check for and cleanly handle when executionContextFactory is not in place in ServletContext attr + if (ecfi == null) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "System is initializing, try again soon.") + return + } + + try { + // Verify the incoming webhook request + verifyIncomingWebhook(request, response, ecfi.getEci()) + chain.doFilter(req, resp) + } catch(Throwable t) { + logger.error("Error occurred in Shopify Webhook verification", t) + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Error in Shopify webhook verification: ${t.toString()}") + } + } + + @Override + void destroy() { + // Your implementa tion here } + } + + void verifyIncomingWebhook(HttpServletRequest request, HttpServletResponse response, ExecutionContextImpl ec) { + + String hmac = request.getHeader("X-Shopify-Hmac-SHA256") + String shopDomain = request.getHeader("X-Shopify-Shop-Domain") + String webhookTopic = request.getHeader("X-Shopify-Topic") + String webhookId = request.getHeader("X-Shopify-Webhook-Id") + + String requestBody = IOUtils.toString(request.getReader()); + if (requestBody.length() == 0) { + logger.warn("The request body for webhook ${webhookTopic} is empty for Shopify ${shopDomain}, cannot verify webhook") + response.sendError(HttpServletResponse.SC_BAD_REQUEST, "The Request Body is empty for Shopify webhook") + return + } + request.setAttribute("payload", ContextJavaUtil.jacksonMapper.readValue(requestBody, Map.class)) + + + EntityList systemMessageRemoteList = ec.entityFacade.find("moqui.service.message.SystemMessageRemote") + .condition("sendUrl", EntityCondition.ComparisonOperator.LIKE, "%"+shopDomain+"%") + .condition("sendSharedSecret", EntityCondition.ComparisonOperator.NOT_EQUAL, null) + .disableAuthz().list() + for (EntityValue systemMessageRemote in systemMessageRemoteList) { + // Call service to verify Hmac + Map result = ec.serviceFacade.sync().name("co.hotwax.shopify.webhook.ShopifyWebhookServices.verify#Hmac") + .parameters([message:requestBody, hmac:hmac, sharedSecret:systemMessageRemote.sendSharedSecret]) + .disableAuthz().call() + // If the hmac matched with the calculatedHmac, break the loop and return + if (result.isValidWebhook) { + request.setAttribute("systemMessageRemoteId", systemMessageRemote.systemMessageRemoteId) + request.setAttribute("webhookId", webhookId) + request.setAttribute("webhookTopic", webhookTopic) + return; + } + } + logger.warn("The webhook ${webhookTopic} HMAC header did not match with the computed HMAC for Shopify ${shopDomain}") + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "HMAC verification failed for Shopify ${shopDomain} for webhook ${webhookTopic}") + } +} \ No newline at end of file diff --git a/template/graphQL/WebhookSubscriptionCreate.ftl b/template/graphQL/WebhookSubscriptionCreate.ftl new file mode 100644 index 0000000..70e90e3 --- /dev/null +++ b/template/graphQL/WebhookSubscriptionCreate.ftl @@ -0,0 +1,35 @@ +<#-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<@compress single_line=true> + mutation { + webhookSubscriptionCreate( + topic: ${topic} + webhookSubscription: { + format: JSON, + callbackUrl: "${endPoint}"} + ) { + userErrors { + field + message + } + webhookSubscription { + id + } + } + } + \ No newline at end of file diff --git a/template/graphQL/WebhookSubscriptionDelete.ftl b/template/graphQL/WebhookSubscriptionDelete.ftl new file mode 100644 index 0000000..96fe3f4 --- /dev/null +++ b/template/graphQL/WebhookSubscriptionDelete.ftl @@ -0,0 +1,28 @@ +<#-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<@compress single_line=true> + mutation webhookSubscriptionDelete($id: ID!) { + webhookSubscriptionDelete(id: $id) { + deletedWebhookSubscriptionId + userErrors { + field + message + } + } + } + \ No newline at end of file diff --git a/template/graphQL/WebhookSubscriptionsQuery.ftl b/template/graphQL/WebhookSubscriptionsQuery.ftl new file mode 100644 index 0000000..09d84e7 --- /dev/null +++ b/template/graphQL/WebhookSubscriptionsQuery.ftl @@ -0,0 +1,37 @@ +<#-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<@compress single_line=true> + query { + webhookSubscriptions(first: 1 + <#if queryParams?has_content && queryParams.topics?has_content>, topics: ${queryParams.topics} + <#if queryParams?has_content && queryParams.callbackUrl?has_content>, callbackUrl: "${queryParams.callbackUrl}") { + edges { + node { + id + topic + endpoint { + __typename + ... on WebhookHttpEndpoint { + callbackUrl + } + } + } + } + } + } + \ No newline at end of file