Add verbose option (#97)

* start verbose option

* start unit test

* finish tests

* remove print

* update examples

* update yaml

examples/main.go

@@ -14,24 +14,38 @@ func main() {
  // You can also create a client with a cache (capacity, page expiration, interval to check expiration of each page)
  // tg = client.NewTableGetter("github.com/atye/wikitable2json/examples", client.WithCache(5, 5*time.Second, 5*time.Second))
 
- matrix, err := tg.GetTablesMatrix(context.Background(), "Arhaan_Khan", "en", false)
+ matrix, err := tg.GetMatrix(context.Background(), "Arhaan_Khan", "en", false)
  if err != nil {
  log.Fatal(err)
  }
 
  fmt.Println(matrix)
 
- matrix, err = tg.GetTablesMatrix(context.Background(), "Arhaan_Khan", "en", false, 0)
+ matrix, err = tg.GetMatrix(context.Background(), "Arhaan_Khan", "en", false, 0)
  if err != nil {
  log.Fatal(err)
  }
 
  fmt.Println(matrix)
 
- keyValue, err := tg.GetTablesKeyValue(context.Background(), "Arhaan_Khan", "en", false, 1, 1)
+ matrixVerbose, err := tg.GetMatrixVerbose(context.Background(), "Arhaan_Khan", "en", false)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ fmt.Println(matrixVerbose)
+
+ keyValue, err := tg.GetKeyValue(context.Background(), "Arhaan_Khan", "en", false, 1, 1)
  if err != nil {
  log.Fatal(err)
  }
 
  fmt.Println(keyValue)
+
+ keyValueVerbose, err := tg.GetKeyValueVerbose(context.Background(), "Arhaan_Khan", "en", false, 1, 1)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ fmt.Println(keyValueVerbose)
 }

internal/entrypoint/entrypoint_test.go

@@ -43,6 +43,8 @@ func TestAPI(t *testing.T) {
  w.Write(getPageBytes(t, "issue77"))
  case "/api/rest_v1/page/html/issue85":
  w.Write(getPageBytes(t, "issue85"))
+ case "/api/rest_v1/page/html/issue93":
+ w.Write(getPageBytes(t, "issue93"))
  case "/api/rest_v1/page/html/reference":
  w.Write(getPageBytes(t, "reference"))
  case "/api/rest_v1/page/html/simpleKeyValue":
@@ -155,6 +157,36 @@ func TestAPI(t *testing.T) {
  }
  })
 
+ t.Run("MatrixVerbose", func(t *testing.T) {
+ tests := []struct {
+ name string
+ url string
+ want [][][]client.Verbose
+ }{
+ {
+ "Issue77",
+ fmt.Sprintf("http://localhost:%s/api/issue77?verbose=true", PORT),
+ Issue77MatrixVerbose,
+ },
+ {
+ "Issue93",
+ fmt.Sprintf("http://localhost:%s/api/issue93?verbose=true", PORT),
+ Issue93MatrixVerbose,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ var got [][][]client.Verbose
+ execGetRequest(t, tc.url, &got)
+
+ if !reflect.DeepEqual(tc.want, got) {
+ t.Errorf("want %v\n got %v", tc.want, got)
+ }
+ })
+ }
+ })
+
  t.Run("KeyValue", func(t *testing.T) {
  tests := []struct {
  name string
@@ -200,6 +232,36 @@ func TestAPI(t *testing.T) {
  }
  })
 
+ t.Run("KeyValueVerbose", func(t *testing.T) {
+ tests := []struct {
+ name string
+ url string
+ want [][]map[string]client.Verbose
+ }{
+ {
+ "Issue77",
+ fmt.Sprintf("http://localhost:%s/api/issue77?keyRows=1&verbose=true", PORT),
+ Issue77KeyValueVerbose,
+ },
+ {
+ "Issue93",
+ fmt.Sprintf("http://localhost:%s/api/issue93?keyRows=1&verbose=true", PORT),
+ Issue93KeyValueVerbose,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ var got [][]map[string]client.Verbose
+ execGetRequest(t, tc.url, &got)
+
+ if !reflect.DeepEqual(tc.want, got) {
+ t.Errorf("want %v\n got %v", tc.want, got)
+ }
+ })
+ }
+ })
+
  t.Run("NoTables", func(t *testing.T) {
  req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%s/api/noTables", PORT), nil)
  if err != nil {

internal/entrypoint/static/dist/openapi.yaml

@@ -19,6 +19,9 @@ info:
  ### Get the second table on page [Arhaan_Khan](https://en.wikipedia.org/wiki/Arhaan_Khan) in a key-value format using the first row as keys:
  [https://www.wikitable2json.com/api/Arhaan_Khan?table=1&keyRows=1](https://www.wikitable2json.com/api/Arhaan_Khan?table=1&keyRows=1)
 
+ ### Get the second table on page [Arhaan_Khan](https://en.wikipedia.org/wiki/Arhaan_Khan) in a key-value format with verbose output using the first row as keys:
+ [https://www.wikitable2json.com/api/Arhaan_Khan?table=1&keyRows=1&verbose=true](https://www.wikitable2json.com/api/Arhaan_Khan?table=1&keyRows=1&verbose=true)
+
  ### Get the second table on page [1970–71_Chester_F.C._season](https://en.wikipedia.org/wiki/1970–71_Chester_F.C._season) in a key-value format using the first two rows as key:
  [https://www.wikitable2json.com/api/1970–71_Chester_F.C._season?table=1&keyRows=2](https://www.wikitable2json.com/api/1970–71_Chester_F.C._season?table=1&keyRows=2)
 
@@ -73,6 +76,14 @@ paths:
  schema:
  type: string
  default: false
+ - name: verbose
+ description: |
+ Set to true to enable verbose output<br/>
+ in: query
+ required: false
+ schema:
+ type: string
+ default: false
  responses:
  "200":
  description: A successful response.
@@ -81,7 +92,9 @@ paths:
  schema:
  oneOf:
  - $ref: "#/components/schemas/matrix"
+ - $ref: "#/components/schemas/matrixVerbose"
  - $ref: "#/components/schemas/keyValue"
+ - $ref: "#/components/schemas/keyValueVerbose"
  default:
  description: An error response.
  content:
@@ -99,10 +112,33 @@ components:
  type: array
  items:
  type: string
+ matrixVerbose:
+ description: List of tables in the default, 2D format with verbose output
+ type: array
+ items:
+ type: array
+ items:
+ type: array
+ items:
+ type: object
+ additionalProperties:
+ type: string
  keyValue:
  description: |
  List of tables in the key-value format with the keys being the first x rows specified in the keyRows query<br/>
  type: array
+ items:
+ type: array
+ items:
+ type: object
+ additionalProperties:
+ type: object
+ additionalProperties:
+ type: string
+ keyValueVerbose:
+ description: |
+ List of tables in the key-value format with the keys being the first x rows specified in the keyRows query with verbose output<br/>
+ type: array
  items:
  type: array
  items:

internal/entrypoint/testdata/issue77.html

@@ -5,8 +5,6 @@
  <th class="unsortable">Beschreibung</th>
  <th style="width:11%;">Akten-Nr.</th>
  <th style="width:121px;" class="unsortable">Bild</th></tr>
- <tr>
- </tr>
  <tr id="ehemals_D-5-74-123-14_zugehörig" class="vcard">
 
  <td data-sort-value="FeuchtHauptstrasse 00037">Feucht<br>Hauptstraße 37<br><small>(<span id="Feucht,_Hauptstraße_37,_Ehemaliges_Wirtschaftsgebäude" class="plainlinks-print" title="Koordinate"><a rel="mw:ExtLink" href="https://geohack.toolforge.org/geohack.php?pagename=Liste_der_Baudenkm%C3%A4ler_in_Feucht&amp;language=de&amp;params=49.37546_N_11.21422_E_region:DE-BY_type:building&amp;title=Feucht%2C+Hauptstra%C3%9Fe+37%2C+Ehemaliges+Wirtschaftsgeb%C3%A4ude" class="external text">Standort</a></span>)</small></td>

internal/entrypoint/testdata/issue93.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html>
+
+<body>
+ <table class="standard">
+ <tbody>
+ <tr>
+ <th>header1</th>
+ <th>header2</th>
+ </tr>
+ <tr>
+ <td>test</td>
+ <td id="mwAts"><span class="flagicon" about="#mwt426" typeof="mw:Transclusion" data-mw="{&quot;parts&quot;:[{&quot;template&quot;:{&quot;target&quot;:{&quot;wt&quot;:&quot;flagdeco&quot;,&quot;href&quot;:&quot;./Template:Flagdeco&quot;},&quot;params&quot;:{&quot;1&quot;:{&quot;wt&quot;:&quot;BOL&quot;}},&quot;i&quot;:0}}]}" id="mwAtw"><span class="mw-image-border" typeof="mw:File"><span><img alt="" resource="./File:Bandera_de_Bolivia_(Estado).svg" src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b3/Bandera_de_Bolivia_%28Estado%29.svg/22px-Bandera_de_Bolivia_%28Estado%29.svg.png" decoding="async" data-file-width="1100" data-file-height="750" data-file-type="drawing" height="15" width="22" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b3/Bandera_de_Bolivia_%28Estado%29.svg/34px-Bandera_de_Bolivia_%28Estado%29.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b3/Bandera_de_Bolivia_%28Estado%29.svg/44px-Bandera_de_Bolivia_%28Estado%29.svg.png 2x" class="mw-file-element"></span></span></span> <a rel="mw:WikiLink" href="./Bolivia" title="Bolivia" id="mwAt0">Bolivia, Plurinational State of</a></td>
+ </tr>
+ </tbody>
+ </table>
+</body>
+
+</html>

internal/entrypoint/types_test.go

@@ -1,5 +1,7 @@
 package entrypoint
 
+import "github.com/atye/wikitable2json/pkg/client"
+
 var (
  GoldenMatrix = [][][]string{
  {
@@ -86,11 +88,81 @@ var (
  Issue77Matrix = [][][]string{
  {
  {"Lage", "Objekt", "Beschreibung", "Akten-Nr.", "Bild"},
- {},
  {"FeuchtHauptstraße 37(Standort)", "Ehemaliges Wirtschaftsgebäude", "Zweigeschossiger Satteldachbau mit Fachwerkobergeschoss, bezeichnet mit „1697“", "ehemals D-5-74-123-14 zugehörig", "weitere Bilder"},
  },
  }
 
+ Issue77MatrixVerbose = [][][]client.Verbose{
+ {
+ {
+ {
+ Text: "Lage",
+ },
+ {
+ Text: "Objekt",
+ },
+ {
+ Text: "Beschreibung",
+ },
+ {
+ Text: "Akten-Nr.",
+ },
+ {
+ Text: "Bild",
+ },
+ },
+ {
+ {
+ Text: "FeuchtHauptstraße 37(Standort)",
+ Links: []string{"https://geohack.toolforge.org/geohack.php?pagename=Liste_der_Baudenkm%C3%A4ler_in_Feucht&language=de&params=49.37546_N_11.21422_E_region:DE-BY_type:building&title=Feucht%2C+Hauptstra%C3%9Fe+37%2C+Ehemaliges+Wirtschaftsgeb%C3%A4ude"},
+ },
+ {
+ Text: "Ehemaliges Wirtschaftsgebäude",
+ },
+ {
+ Text: "Zweigeschossiger Satteldachbau mit Fachwerkobergeschoss, bezeichnet mit „1697“",
+ },
+ {
+ Text: "ehemals D-5-74-123-14 zugehörig",
+ },
+ {
+ Text: "weitere Bilder",
+ Links: []string{
+ "Datei:2018_Feucht_Hauptstraße_37_02.jpg",
+ "https://commons.wikimedia.org/wiki/Category:Hauptstraße%2037%20(Ehemaliges%20Wirtschaftsgebäude,%20D-5-74-123-14)",
+ },
+ },
+ },
+ },
+ }
+
+ Issue77KeyValueVerbose = [][]map[string]client.Verbose{
+ {
+ {
+ "Lage": {
+ Text: "FeuchtHauptstraße 37(Standort)",
+ Links: []string{"https://geohack.toolforge.org/geohack.php?pagename=Liste_der_Baudenkm%C3%A4ler_in_Feucht&language=de&params=49.37546_N_11.21422_E_region:DE-BY_type:building&title=Feucht%2C+Hauptstra%C3%9Fe+37%2C+Ehemaliges+Wirtschaftsgeb%C3%A4ude"},
+ },
+ "Objekt": {
+ Text: "Ehemaliges Wirtschaftsgebäude",
+ },
+ "Beschreibung": {
+ Text: "Zweigeschossiger Satteldachbau mit Fachwerkobergeschoss, bezeichnet mit „1697“",
+ },
+ "Akten-Nr.": {
+ Text: "ehemals D-5-74-123-14 zugehörig",
+ },
+ "Bild": {
+ Text: "weitere Bilder",
+ Links: []string{
+ "Datei:2018_Feucht_Hauptstraße_37_02.jpg",
+ "https://commons.wikimedia.org/wiki/Category:Hauptstraße%2037%20(Ehemaliges%20Wirtschaftsgebäude,%20D-5-74-123-14)",
+ },
+ },
+ },
+ },
+ }
+
  Issue85KeyValue = [][]map[string]string{
  {
  {
@@ -186,4 +258,40 @@ var (
  },
  },
  }
+
+ Issue93MatrixVerbose = [][][]client.Verbose{
+ {
+ {
+ {
+ Text: "header1",
+ },
+ {
+ Text: "header2",
+ },
+ },
+ {
+ {
+ Text: "test",
+ },
+ {
+ Text: "Bolivia, Plurinational State of",
+ Links: []string{"Bolivia"},
+ },
+ },
+ },
+ }
+
+ Issue93KeyValueVerbose = [][]map[string]client.Verbose{
+ {
+ {
+ "header1": {
+ Text: "test",
+ },
+ "header2": {
+ Text: "Bolivia, Plurinational State of",
+ Links: []string{"Bolivia"},
+ },
+ },
+ },
+ }
 )