Merge pull request #73 from Swirrl/circleci

Port to circleci and tools.build
Swirrl · Feb 21, 2022 · c1faf87 · c1faf87
2 parents f43f76a + 79cb5f6
commit c1faf87
Show file tree

Hide file tree

Showing 12 changed files with 242 additions and 47 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,65 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
+jobs:
+  test:
+    docker:
+      - image: cimg/clojure:1.10
+        auth:
+          username: $DOCKERHUB_USERNAME # can specify string literal values
+          password: $DOCKERHUB_PASSWORD # or project environment variable reference
+    # Add steps to the job
+    # See: https://circleci.com/docs/2.0/configuration-reference/#steps
+    steps:
+      - checkout
+      - run:
+          name: Test
+          command: "clojure -T:build test"
+      - store_test_results:
+          path: test-results
+
+  deploy:
+    docker:
+      - image: cimg/clojure:1.10
+        auth:
+          username: $DOCKERHUB_USERNAME # can specify string literal values
+          password: $DOCKERHUB_PASSWORD # or project environment variable reference
+    # Add steps to the job
+    # See: https://circleci.com/docs/2.0/configuration-reference/#steps
+    steps:
+      - checkout
+      - run:
+          name: Build
+          command: clojure -T:build build-lib
+      - run:
+          name: Deploy
+          command: |
+            if [ -n "${CIRCLE_TAG}" ]; then
+               clojure -T:build deploy
+            else
+               /bin/echo "Not deploying, not a tagged commit"
+            fi
+
+# Invoke jobs via workflows
+# See: https://circleci.com/docs/2.0/configuration-reference/#workflows
+workflows:
+  build-workflow:
+    jobs:
+      - test:
+          context:
+            - swirrl-dockerhub-consumer
+          filters:
+            tags:
+              only: /^v.*/
+      - deploy:
+          context:
+            - swirrl-dockerhub-consumer
+            - swirrl-clojars-publisher
+          requires:
+            - test
+          filters:
+            tags:
+              only: /^v.*/
diff --git a/.gitignore b/.gitignore
@@ -1,8 +1,6 @@
 /target
 /classes
 /checkouts
-pom.xml
-pom.xml.asc
 *.jar
 *.class
 /.lein-*
@@ -14,3 +12,4 @@ csv2rdf.iml
 .cpcache/
 /.lsp/
 /.clj-kondo/.cache
+/test-results/
diff --git a/.travis.yml b/.travis.yml
diff --git a/README.md b/README.md
@@ -1,24 +1,16 @@
 # csv2rdf
 
-[![Build Status](https://travis-ci.org/Swirrl/csv2rdf.svg?branch=master)](https://travis-ci.org/Swirrl/csv2rdf)
+[![CircleCI](https://circleci.com/gh/Swirrl/csv2rdf/tree/master.svg?style=svg)](https://circleci.com/gh/Swirrl/csv2rdf/tree/master)
 
-Command line application (and clojure library) for converting [CSV to RDF](https://www.w3.org/TR/2015/REC-csv2rdf-20151217/) according to the specifications for [CSV on the web](https://w3c.github.io/csvw/)
+Command line application (and clojure library) for converting [CSV to RDF](https://www.w3.org/TR/2015/REC-csv2rdf-20151217/) according to the specifications for [CSV on the web](https://w3c.github.io/csvw/).
 
-## Native Builds 
+## Native Builds
 
 We have some experimental native builds of the command line app here:
 
 - For [linux x86](https://github.com/Swirrl/csv2rdf/releases/tag/graal-linux-0.4.7-SNAPSHOT-c8fe70c)
 - For [mac os](https://github.com/Swirrl/csv2rdf/releases/tag/graal-0.4.7-SNAPSHOT-8839f3f)
 
-## Building
-
-Standalone JARs can be built using [leiningen](https://leiningen.org/). After installing leiningen an uberjar can be built by running the following in the project root directory:
-
-    lein uberjar
-
-This will create a `target/csv2rdf-VERSION-standalone.jar` which can be run.
-
 ## Running
 
 csv2rdf can be run from the command line given the location of either a tabular data file or metadata file referencing the described tabular file. The location
@@ -62,9 +54,8 @@ need to be provided when processing from a metadata file since the metadata shou
 
 csv2rdf also exposes its functionality as a library - please see [the csv2rdf library](doc/library.md) for a description of the library and its interface.
 
-## Overview of the code
-
-See [overview of the code](doc/code.md) for an overview of the codebase.
+- See [overview of the code](doc/code.md) for an overview of the codebase.
+- See [Developing csv2rdf itself](doc/developing.md) for a quickstart guide on how to work on the library and application itself.
 
 ## License
 

diff --git a/bin/kaocha b/bin/kaocha
@@ -1,2 +1,2 @@
 #!/usr/bin/env sh
-clojure -A:with-logging:test:dev -m kaocha.runner "$@"
+clojure -M:with-logging:test:dev "$@"
diff --git a/build.clj b/build.clj
@@ -0,0 +1,65 @@
+(ns build
+  (:require [clojure.tools.build.api :as b]
+            [org.corfield.build :as bb]
+            [clojure.string :as str])
+  (:refer-clojure :exclude [test]))
+
+(def lib 'swirrl/csv2rdf)
+(def version (str/replace (or (System/getenv "CIRCLE_TAG")
+                              "v0.5.999-SNAPSHOT")
+                          #"^v" ""))
+(def class-dir "target/classes")
+
+
+(def jar-file (format "target/%s-%s.jar" (name lib) version))
+
+(defn test
+  "Run the tests"
+  [opts]
+  (bb/run-tests (assoc opts :aliases [:with-logging :dev])))
+
+(defn build-lib
+  "Run the CI pipeline of tests (and build the JAR)."
+  [opts]
+  (-> opts
+      (assoc :lib lib
+             :version version
+             :src-pom "template/pom.xml")
+      (bb/clean)
+      (bb/jar)))
+
+(defn install
+  "Install the JAR locally."
+  [opts]
+  (-> opts
+      (assoc :lib lib :version version)
+      (bb/install)))
+
+(defn deploy
+  "Deploy the JAR to Clojars.
+
+  NOTE: this expects a tag to be set; typically done via the github release UI."
+  [opts]
+  (-> opts
+      (assoc :lib lib :version version)
+      (bb/deploy)))
+
+(defn clean [_]
+  (b/delete {:path "target"}))
+
+;; NOTE this is not yet built by CI
+(defn build-app
+  "Build an uberjar for the command line app"
+  [_]
+  (clean nil)
+  (b/copy-dir {:src-dirs ["resources"]
+               :target-dir class-dir})
+  (let [basis (b/create-basis {:aliases [:cli :with-logging]})]
+    (b/compile-clj {:basis basis
+                    :class-dir class-dir
+                    :src-dirs ["src"]})
+    (bb/uber {:main 'csv2rdf.main
+              :basis basis
+              :class-dir class-dir
+              :uber-file "target/csv2rdf-app.jar"
+              })))
diff --git a/deps.edn b/deps.edn
@@ -1,9 +1,9 @@
 {:deps {org.clojure/clojure                {:mvn/version "1.10.3"}
         org.clojure/data.json              {:mvn/version "2.4.0"}
-        grafter                            {:mvn/version "2.1.7"}
+        grafter/grafter                    {:mvn/version "2.1.7"}
         com.github.java-json-tools/uri-template {:mvn/version "0.10"}
         org.apache.httpcomponents/httpcore {:mvn/version "4.4.15"}
-        clj-http                           {:mvn/version "3.7.0"}
+        clj-http/clj-http                  {:mvn/version "3.7.0"}
         org.clojure/tools.cli              {:mvn/version "1.0.206"}
         org.clojure/tools.logging          {:mvn/version "1.2.4"}
         org.slf4j/slf4j-api                {:mvn/version "1.7.36"}}
@@ -23,6 +23,12 @@
            :test
            {:extra-deps
             {lambdaisland/kaocha {:mvn/version "1.63.998"}
+             lambdaisland/kaocha-junit-xml {:mvn/version "0.0.76"}
              org.clojure/test.check {:mvn/version "1.1.1"}
              org.clojure/data.csv {:mvn/version "1.0.0"}}
-            :extra-paths ["test" "test/resources"]}}}
+            :extra-paths ["test" "test/resources"]
+            :main-opts ["-m" "kaocha.runner"]}
+
+           :build {:deps {io.github.seancorfield/build-clj
+                          {:git/tag "v0.6.3" :git/sha "9b8e09b"}}
+                   :ns-default build}}}
diff --git a/doc/developing.md b/doc/developing.md
@@ -0,0 +1,70 @@
+# Developing csv2rdf itself
+
+These notes cover how to use the tooling to build, test and develop
+the `csv2rdf` library and command line app.
+
+## Experimental GraalVM builds
+
+These are not yet merged to master; see the [`graal-native-image`](https://github.com/Swirrl/csv2rdf/pull/54) PR.
+
+## tools.deps and tools.build
+
+`csv2rdf` is now maintained with the tools.deps and tools.build libraries.
+
+The best workflow is always to use a REPL, exactly how this is done
+largely depends on your editor environment. However a basic REPL can
+be started with:
+
+```
+$ clj
+```
+
+or via something like a custom `:cider` alias in your `~/.clojure/deps.edn`
+
+```
+$ clj -M:cider
+```
+
+To run the tests:
+
+```
+$ clojure -T:build test
+```
+
+To build the command line application as an uberjar:
+
+```
+$ clojure -T:build build-app
+```
+
+and to run that app:
+
+```
+$ java -jar target/csv2rdf-app.jar
+16:43:56.344 [main] ERROR csv2rdf.main - User metadata or tabular data file required
+Usage:
+  -t, --tabular TABULAR                    Location of the tabular file
+  -u, --user-metadata METADATA             Location of the metadata file
+  -o, --output-file OUTPUT                 Output file to write to
+  -m, --mode MODE               :standard  CSVW mode to run
+```
+
+## Circle CI
+
+Circle CI will run the tests on push for each branch.
+
+Currently CI only builds and deploys the library, releases of the
+command line app need to be made by hand.
+
+Releases are listed on the github [releases page](https://github.com/Swirrl/csv2rdf/releases).
+
+### Deploying new library jars to clojars
+
+You can do this by tagging a commit with `vX.Y.Z`, which will trigger
+the tests to be run by CI; if they then pass a build will pushed to
+clojars with that version number.
+
+A good workflow for this is to use github releases to coin a new
+release document the changes, and create that tag. Be sure to prefix
+your tags with `v` from now on (as some clojure tooling requires it
+e.g. cljdocs).
diff --git a/project.clj b/project.clj
diff --git a/template/pom.xml b/template/pom.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>swirrl</groupId>
+  <artifactId>csv2rdf</artifactId>
+  <packaging>jar</packaging>
+  <name>csv2rdf</name>
+  <description>A clojure API implementing the W3C CSVW csv2rdf processor</description>
+  <url>https://github.com/Swirrl/csv2rdf</url>
+  <licenses>
+    <license>
+      <name>Eclipse Public License</name>
+      <url>http://www.eclipse.org/legal/epl-v10.html</url>
+    </license>
+  </licenses>
+  <scm>
+    <url>https://github.com/Swirrl/csv2rdf</url>
+    <connection>scm:git:git://github.com/Swirrl/csv2rdf.git</connection>
+    <developerConnection>scm:git:ssh://git@github.com/Swirrl/csv2rdf.git</developerConnection>
+  </scm>
+</project>
diff --git a/tests.edn b/tests.edn
@@ -0,0 +1,6 @@
+#kaocha/v1
+{:plugins [:kaocha.plugin/profiling :kaocha.plugin/junit-xml]
+ :kaocha.plugin.junit-xml/target-file "test-results/kaocha/results.xml"
+ :reporter kaocha.report.progress/progress
+ ;; :plugins [kaocha.plugin/profiling]
+ }
diff --git a/travis/install_clojure.sh b/travis/install_clojure.sh