Merge pull request #75 from mdn1023/bugfix-cojp-domain-parser

(bug-fix): enable correct parsing of second level jp domains

parser_test.go

@@ -142,7 +142,11 @@ func TestParse(t *testing.T) {
  }
 
  if !assert.IsContains([]string{"aq", "au", "de", "eu", "gov", "hm", "name", "nl", "nz", "ir", "tk",
- "xn--mgba3a4f16a"}, extension) {
+ "xn--mgba3a4f16a"}, extension) &&
+ !strings.Contains(domain, "ac.jp") &&
+ !strings.Contains(domain, "co.jp") &&
+ !strings.Contains(domain, "go.jp") &&
+ !strings.Contains(domain, "ne.jp") {
  assert.NotZero(t, whoisInfo.Domain.CreatedDate)
  assert.NotNil(t, whoisInfo.Domain.CreatedDateInTime)
  }
@@ -154,7 +158,11 @@ func TestParse(t *testing.T) {
  }
 
  if !assert.IsContains([]string{"", "aq", "au", "br", "ch", "de", "eu", "gov", "ee",
- "hm", "int", "name", "nl", "nz", "tk", "kz", "hu"}, extension) && !strings.Contains(domain, "co.jp") {
+ "hm", "int", "name", "nl", "nz", "tk", "kz", "hu"}, extension) &&
+ !strings.Contains(domain, "ac.jp") &&
+ !strings.Contains(domain, "co.jp") &&
+ !strings.Contains(domain, "go.jp") &&
+ !strings.Contains(domain, "ne.jp") {
  assert.NotZero(t, whoisInfo.Domain.ExpirationDate)
  assert.NotNil(t, whoisInfo.Domain.ExpirationDateInTime)
  }

prepare.go

@@ -712,6 +712,7 @@ func prepareJP(text string) string {
  if strings.ToLower(token) == "registrant" {
  v = fmt.Sprintf("registrant name: %s", vs[1])
  }
+ v = prepareSecondLevelJP(v, token, vs[1])
  } else {
  if token == addressToken {
  result += ", " + v
@@ -724,6 +725,29 @@ func prepareJP(text string) string {
  return result
 }
 
+// prepareJP prepares specific mappings for second level .jp domains
+// examples include:
+// - co.jp
+// - ac.jp
+// - go.jp
+// - or.jp
+// - ad.jp
+// - ne.jp
+// - gr.jp
+// - ed.jp
+func prepareSecondLevelJP(original string, token string, value string) string {
+ if strings.ToLower(token) == "administrative contact" {
+ return fmt.Sprintf("Administrative Contact ID: %s", strings.TrimSpace(value))
+ }
+ if strings.ToLower(token) == "technical contact" {
+ return fmt.Sprintf("Technical Contact ID: %s", strings.TrimSpace(value))
+ }
+ if strings.ToLower(token) == "organization" || strings.ToLower(token) == "network service name" {
+ return fmt.Sprintf("Registrant Organization: %s", strings.TrimSpace(value))
+ }
+ return original
+}
+
 // prepareUK do prepare the .uk domain
 func prepareUK(text string) string {
  tokens := map[string]string{

testdata/noterror/README.md

@@ -104,8 +104,11 @@ If there is any problem, please feel free to open a new issue.
 | .jobs | [google.jobs](jobs_google.jobs) | [google.jobs](jobs_google.jobs.json) | √ |
 | .jobs | [ybs.jobs](jobs_ybs.jobs) | [ybs.jobs](jobs_ybs.jobs.json) | √ |
 | .jp | [git.jp](jp_git.jp) | [git.jp](jp_git.jp.json) | √ |
+| .jp | [goo.ne.jp](jp_goo.ne.jp) | [goo.ne.jp](jp_goo.ne.jp.json) | √ |
 | .jp | [google.co.jp](jp_google.co.jp) | [google.co.jp](jp_google.co.jp.json) | √ |
 | .jp | [google.jp](jp_google.jp) | [google.jp](jp_google.jp.json) | √ |
+| .jp | [mod.go.jp](jp_mod.go.jp) | [mod.go.jp](jp_mod.go.jp.json) | √ |
+| .jp | [titech.ac.jp](jp_titech.ac.jp) | [titech.ac.jp](jp_titech.ac.jp.json) | √ |
 | .kr | [git.kr](kr_git.kr) | [git.kr](kr_git.kr.json) | √ |
 | .kr | [google.kr](kr_google.kr) | [google.kr](kr_google.kr.json) | √ |
 | .kz | [google.kz](kz_google.kz) | [google.kz](kz_google.kz.json) | √ |

testdata/noterror/jp_goo.ne.jp

@@ -0,0 +1,26 @@
+[ JPRS database provides information on network administration. Its use is ]
+[ restricted to network administration purposes. For further information, ]
+[ use 'whois -h whois.jprs.jp help'. To suppress Japanese output, add'/e' ]
+[ at the end of command, e.g. 'whois -h whois.jprs.jp xxx/e'. ]
+[ ]
+[ Notice -------------------------------------------------------------------- ]
+[ JPRS will add the [Lock Status] element to the response format of JP domain ]
+[ name on November 12, 2023. ]
+[ For further information, please see the following webpage. ]
+[ https://jprs.jp/whatsnew/notice/2023/231112.html (only in Japanese) ]
+[ --------------------------------------------------------------------------- ]
+Domain Information:
+a. [Domain Name] GOO.NE.JP
+d. [Network Service Name] GOO
+l. [Organization Type] Network Service
+m. [Administrative Contact] MS57072JP
+n. [Technical Contact] TH53991JP
+p. [Name Server] ns1.goo.ne.jp
+p. [Name Server] ns2.goo.ne.jp
+p. [Name Server] ns.intervia.ad.jp
+p. [Name Server] ns.via.or.jp
+s. [Signing Key] 
+[State] Connected (2024/06/30)
+[Registered Date] 2004/06/15
+[Connected Date] 2004/06/15
+[Last Update] 2023/07/31 12:30:39 (JST)

testdata/noterror/jp_goo.ne.jp.json

@@ -0,0 +1,28 @@
+{
+ "domain": {
+ "domain": "goo.ne.jp",
+ "punycode": "goo.ne.jp",
+ "name": "goo.ne",
+ "extension": "jp",
+ "status": [
+ "connected"
+ ],
+ "name_servers": [
+ "ns1.goo.ne.jp",
+ "ns2.goo.ne.jp",
+ "ns.intervia.ad.jp",
+ "ns.via.or.jp"
+ ],
+ "created_date": "2004/06/15",
+ "updated_date": "2023/07/31 12:30:39 (JST)"
+ },
+ "registrant": {
+ "organization": "GOO"
+ },
+ "administrative": {
+ "id": "MS57072JP"
+ },
+ "technical": {
+ "id": "TH53991JP"
+ }
+}

testdata/noterror/jp_goo.ne.jp.pre

@@ -0,0 +1,26 @@
+[ JPRS database provides information on network administration. Its use is ]
+restricted to network administration purposes. For further information, :
+use 'whois -h whois.jprs.jp help'. To suppress Japanese output, add'/e' :
+at the end of command, e.g. 'whois -h whois.jprs.jp xxx/e'. :
+:
+Notice -------------------------------------------------------------------- :
+JPRS will add the [Lock Status: element to the response format of JP domain ]
+name on November 12, 2023. :
+For further information, please see the following webpage. :
+https://jprs.jp/whatsnew/notice/2023/231112.html (only in Japanese) :
+--------------------------------------------------------------------------- :
+Domain Information:
+Domain Name: GOO.NE.JP
+Registrant Organization: GOO
+Organization Type: Network Service
+Administrative Contact ID: MS57072JP
+Technical Contact ID: TH53991JP
+Name Server: ns1.goo.ne.jp
+Name Server: ns2.goo.ne.jp
+Name Server: ns.intervia.ad.jp
+Name Server: ns.via.or.jp
+Signing Key:
+State: Connected (2024/06/30)
+Registered Date: 2004/06/15
+Connected Date: 2004/06/15
+Last Update: 2023/07/31 12:30:39 (JST)

testdata/noterror/jp_google.co.jp.json

@@ -16,10 +16,13 @@
  "created_date": "2001/03/22",
  "updated_date": "2023/04/01 01:05:57 (JST)"
  },
+ "registrant": {
+ "organization": "Google Japan G.K."
+ },
  "administrative": {
- "name": "YN47525JP"
+ "id": "YN47525JP"
  },
  "technical": {
- "name": "SH36113JP"
+ "id": "SH36113JP"
  }
 }

testdata/noterror/jp_google.co.jp.pre

@@ -11,10 +11,10 @@ https://jprs.jp/whatsnew/notice/2023/231112.html (only in Japanese) :
 --------------------------------------------------------------------------- :
 Domain Information:
 Domain Name: GOOGLE.CO.JP
-Organization: Google Japan G.K.
+Registrant Organization: Google Japan G.K.
 Organization Type: GK
-Administrative Contact: YN47525JP
-Technical Contact: SH36113JP
+Administrative Contact ID: YN47525JP
+Technical Contact ID: SH36113JP
 Name Server: ns1.google.com
 Name Server: ns2.google.com
 Name Server: ns3.google.com

testdata/noterror/jp_mod.go.jp

@@ -0,0 +1,33 @@
+[ JPRS database provides information on network administration. Its use is ]
+[ restricted to network administration purposes. For further information, ]
+[ use 'whois -h whois.jprs.jp help'. To suppress Japanese output, add'/e' ]
+[ at the end of command, e.g. 'whois -h whois.jprs.jp xxx/e'. ]
+[ ]
+[ Notice -------------------------------------------------------------------- ]
+[ JPRS will add the [Lock Status] element to the response format of JP domain ]
+[ name on November 12, 2023. ]
+[ For further information, please see the following webpage. ]
+[ https://jprs.jp/whatsnew/notice/2023/231112.html (only in Japanese) ]
+[ --------------------------------------------------------------------------- ]
+Domain Information:
+a. [Domain Name] MOD.GO.JP
+g. [Organization] Ministry of Defense
+l. [Organization Type] Government Office
+m. [Administrative Contact] HM15693JP
+n. [Technical Contact] HM15693JP
+p. [Name Server] ns1.mod.go.jp
+p. [Name Server] ns2.mod.go.jp
+p. [Name Server] ns3.mod.go.jp
+p. [Name Server] ns5.mod.go.jp
+p. [Name Server] ns6.mod.go.jp
+p. [Name Server] ns6-tk01.ocn.ad.jp
+p. [Name Server] auth1.ns.gin.ntt.net
+p. [Name Server] auth2.ns.gin.ntt.net
+p. [Name Server] auth3.ns.gin.ntt.net
+p. [Name Server] auth4.ns.gin.ntt.net
+p. [Name Server] auth5.ns.gin.ntt.net
+s. [Signing Key] 
+[State] Connected (2024/12/31)
+[Registered Date] 2006/12/19
+[Connected Date] 2006/12/25
+[Last Update] 2024/01/01 01:04:32 (JST)

testdata/noterror/jp_mod.go.jp.json

@@ -0,0 +1,35 @@
+{
+ "domain": {
+ "domain": "mod.go.jp",
+ "punycode": "mod.go.jp",
+ "name": "mod.go",
+ "extension": "jp",
+ "status": [
+ "connected"
+ ],
+ "name_servers": [
+ "ns1.mod.go.jp",
+ "ns2.mod.go.jp",
+ "ns3.mod.go.jp",
+ "ns5.mod.go.jp",
+ "ns6.mod.go.jp",
+ "ns6-tk01.ocn.ad.jp",
+ "auth1.ns.gin.ntt.net",
+ "auth2.ns.gin.ntt.net",
+ "auth3.ns.gin.ntt.net",
+ "auth4.ns.gin.ntt.net",
+ "auth5.ns.gin.ntt.net"
+ ],
+ "created_date": "2006/12/19",
+ "updated_date": "2024/01/01 01:04:32 (JST)"
+ },
+ "registrant": {
+ "organization": "Ministry of Defense"
+ },
+ "administrative": {
+ "id": "HM15693JP"
+ },
+ "technical": {
+ "id": "HM15693JP"
+ }
+}

testdata/noterror/jp_mod.go.jp.pre

@@ -0,0 +1,33 @@
+[ JPRS database provides information on network administration. Its use is ]
+restricted to network administration purposes. For further information, :
+use 'whois -h whois.jprs.jp help'. To suppress Japanese output, add'/e' :
+at the end of command, e.g. 'whois -h whois.jprs.jp xxx/e'. :
+:
+Notice -------------------------------------------------------------------- :
+JPRS will add the [Lock Status: element to the response format of JP domain ]
+name on November 12, 2023. :
+For further information, please see the following webpage. :
+https://jprs.jp/whatsnew/notice/2023/231112.html (only in Japanese) :
+--------------------------------------------------------------------------- :
+Domain Information:
+Domain Name: MOD.GO.JP
+Registrant Organization: Ministry of Defense
+Organization Type: Government Office
+Administrative Contact ID: HM15693JP
+Technical Contact ID: HM15693JP
+Name Server: ns1.mod.go.jp
+Name Server: ns2.mod.go.jp
+Name Server: ns3.mod.go.jp
+Name Server: ns5.mod.go.jp
+Name Server: ns6.mod.go.jp
+Name Server: ns6-tk01.ocn.ad.jp
+Name Server: auth1.ns.gin.ntt.net
+Name Server: auth2.ns.gin.ntt.net
+Name Server: auth3.ns.gin.ntt.net
+Name Server: auth4.ns.gin.ntt.net
+Name Server: auth5.ns.gin.ntt.net
+Signing Key:
+State: Connected (2024/12/31)
+Registered Date: 2006/12/19
+Connected Date: 2006/12/25
+Last Update: 2024/01/01 01:04:32 (JST)

testdata/noterror/jp_titech.ac.jp

@@ -0,0 +1,27 @@
+[ JPRS database provides information on network administration. Its use is ]
+[ restricted to network administration purposes. For further information, ]
+[ use 'whois -h whois.jprs.jp help'. To suppress Japanese output, add'/e' ]
+[ at the end of command, e.g. 'whois -h whois.jprs.jp xxx/e'. ]
+[ ]
+[ Notice -------------------------------------------------------------------- ]
+[ JPRS will add the [Lock Status] element to the response format of JP domain ]
+[ name on November 12, 2023. ]
+[ For further information, please see the following webpage. ]
+[ https://jprs.jp/whatsnew/notice/2023/231112.html (only in Japanese) ]
+[ --------------------------------------------------------------------------- ]
+Domain Information:
+a. [Domain Name] TITECH.AC.JP
+g. [Organization] Tokyo Institute of Technology
+l. [Organization Type] National University Corporation
+m. [Administrative Contact] YS12912JP
+n. [Technical Contact] MT47768JP
+n. [Technical Contact] YS12912JP
+n. [Technical Contact] NM23856JP
+p. [Name Server] ns.fujisawa.wide.ad.jp
+p. [Name Server] ns1.noc.titech.ac.jp
+p. [Name Server] ns2.noc.titech.ac.jp
+s. [Signing Key] 
+[State] Connected (2024/03/31)
+[Registered Date] 
+[Connected Date] 
+[Last Update] 2023/04/01 01:04:55 (JST)

testdata/noterror/jp_titech.ac.jp.json

@@ -0,0 +1,26 @@
+{
+ "domain": {
+ "domain": "titech.ac.jp",
+ "punycode": "titech.ac.jp",
+ "name": "titech.ac",
+ "extension": "jp",
+ "status": [
+ "connected"
+ ],
+ "name_servers": [
+ "ns.fujisawa.wide.ad.jp",
+ "ns1.noc.titech.ac.jp",
+ "ns2.noc.titech.ac.jp"
+ ],
+ "updated_date": "2023/04/01 01:04:55 (JST)"
+ },
+ "registrant": {
+ "organization": "Tokyo Institute of Technology"
+ },
+ "administrative": {
+ "id": "YS12912JP"
+ },
+ "technical": {
+ "id": "NM23856JP"
+ }
+}

testdata/noterror/jp_titech.ac.jp.pre

@@ -0,0 +1,27 @@
+[ JPRS database provides information on network administration. Its use is ]
+restricted to network administration purposes. For further information, :
+use 'whois -h whois.jprs.jp help'. To suppress Japanese output, add'/e' :
+at the end of command, e.g. 'whois -h whois.jprs.jp xxx/e'. :
+:
+Notice -------------------------------------------------------------------- :
+JPRS will add the [Lock Status: element to the response format of JP domain ]
+name on November 12, 2023. :
+For further information, please see the following webpage. :
+https://jprs.jp/whatsnew/notice/2023/231112.html (only in Japanese) :
+--------------------------------------------------------------------------- :
+Domain Information:
+Domain Name: TITECH.AC.JP
+Registrant Organization: Tokyo Institute of Technology
+Organization Type: National University Corporation
+Administrative Contact ID: YS12912JP
+Technical Contact ID: MT47768JP
+Technical Contact ID: YS12912JP
+Technical Contact ID: NM23856JP
+Name Server: ns.fujisawa.wide.ad.jp
+Name Server: ns1.noc.titech.ac.jp
+Name Server: ns2.noc.titech.ac.jp
+Signing Key:
+State: Connected (2024/03/31)
+Registered Date:
+Connected Date:
+Last Update: 2023/04/01 01:04:55 (JST)