From b4415448675f6f8bfcaf143fafb254d3d3d5e6b0 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sun, 19 Nov 2023 07:27:21 +0000 Subject: [PATCH 01/93] first commit --- README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..94a0c6e --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# backend-flask-server From c854291576b3e99a9aa770557d8b2f5505ee9788 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sun, 19 Nov 2023 07:30:42 +0000 Subject: [PATCH 02/93] add hello world app --- main.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 main.py diff --git a/main.py b/main.py new file mode 100644 index 0000000..305195b --- /dev/null +++ b/main.py @@ -0,0 +1,9 @@ +from flask import Flask +app = Flask(__name__) + +@app.route('/') +def hello_world(): + return 'Hello, World!' + +if __name__ == '__main__': + app.run(debug=True) From 2928e9bcc20d05541aa3532fe727c35e8bfce8e9 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sun, 19 Nov 2023 12:29:23 +0000 Subject: [PATCH 03/93] add ddns model and its unittest --- main.py | 2 +- models/.ddns.py.swp | Bin 0 -> 12288 bytes models/ddns.py | 73 ++++++++++++++++++++++++++++++++++++++++++++ tests/__init__.py | 0 tests/test_ddns.py | 31 +++++++++++++++++++ 5 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 models/.ddns.py.swp create mode 100644 models/ddns.py create mode 100644 tests/__init__.py create mode 100644 tests/test_ddns.py diff --git a/main.py b/main.py index 305195b..783acc5 100644 --- a/main.py +++ b/main.py @@ -3,7 +3,7 @@ @app.route('/') def hello_world(): - return 'Hello, World!' + return 'Hello, World.' if __name__ == '__main__': app.run(debug=True) diff --git a/models/.ddns.py.swp b/models/.ddns.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..76c507a6aaf15712c1192b1640285b2dcb8743aa GIT binary patch literal 12288 zcmeHNzi%8x6rLayAaN2z8Y(ndWZhbv_iRuKjE*WX7>Ovx=4S$@z1_L<%KIxbyKyci zAn_kSC@6p+3K|ep2nlF`0zr`)DF|8=iU^4=d~bI5Zr465C=x})jPmWy&3p6Bd*8g7 zla=c)ys+{tf81GMX!kSr;?FOOU*Fo#x;O7;VGz^kEn7VD^7%;2N3P0*+}r&P5OzxE zP7t|6p*n-mRhu0*O*>KIi%@lZKUPjU+MC5#i~>f1+f(2Wd+gCgebgVej`5=pJb!yM zW|mRFC}0#Y3K#{90!9I&fKk9Ga7QYTrTf^Mi2SZH`jzs%Z^yg*XkJDEqkvJsC}0#Y z3K#{90!9I&fKk9GU=%P4+<^+XK4S;>GxpRy2p<3cU;X`m{Rm^%fFFQwfX{$Wfh)j= zz&pU(zzBE+Xah%qKMphY9q=viG0+EG;2GcuZ~*xG5MwuhUw|KhuYga0*MU`F2KaM^ zv5$Z&fCQF-!@&0k8T%6023`YZfd_%V4ls5V_yYJGco(<~oB_JPL%_}Z82cUg4fq=P z0C*qp0S}l1uHVbpPrzm1NuUeN05_n|Rp33~X@K-v223*w7zK<1MuGpg0=(9g2nWu& zT;#$@^USvL)ORz%@$)u;*Q_f3S$vk;eiFGs+~!hv*(eole%=jpF*nC*bh@j@d6&1W zS!J~_RS!5)q@n8xyJfZ7ymi)UHD?BcikWx2+**HW-C7z`!&ofOkHJaIxx~g)6yhFPyS}gUEuN~8b(V#@SepqD@qeE>cVmAnlomse6btVV2l@O9 zaoQZYVY`mc)l}+b8&bHwqcT@!y;y9O16pRM*V~doh7!o~G!!;!fKm3;Ceng(l?uzA z$fS+2DvPXJIQM#?o5$V;{FtVwSS}{9pn)!@9-}U`f@Dp|^FqQF7d@tJm3dQ)mV;2h zT=tsgFc}Up8Rnv=tyE6EUJwUauUA%6VQp6=?P{^Mj;O8|t)2yua8xKnY9FuVLMCd` z4V2&~FL)x&f+V)ZM3=&W>u7>_=xn(%#(SrW&~qmg+*Mpm4hgZaimLu@$c`=bgV@m< zX!k88V*hMx?T&>A{9rJENDHjaMtwSuWTRqivZqNH+H-V-Phr!V!O=`ck7 zi+(<|Epb73d4@E(mqZcNTjJYT>5}CP5*fLfjSpQ~dhQ8Ur>XkohAd7sHp8lKGtu%# zxJ4W7h}gr=m+8<}to1i}mZBA63`JHu!IkUQ1|2WGV&Qg91s6fu-0)0}CIiKkb(znTHZRh;4QsCO^no-ewGhxQIcnL4XJUhJEvDppKNZ8-bqZ^<<@Sn_45Jk0o)dv zhBK||{LmNg{3f1SnDT0Tb>f}Ug<6j#NkweyRf^3iZ|ZV)n1^4)D}Lg{sWp^8uH$}! q8?MV284IE`ky&-2RUhewWCNK*To(;5y;kXKqVg%fXsVBsXxKjigQyk& literal 0 HcmV?d00001 diff --git a/models/ddns.py b/models/ddns.py new file mode 100644 index 0000000..7e22542 --- /dev/null +++ b/models/ddns.py @@ -0,0 +1,73 @@ +import subprocess +import _thread +from queue import Queue +import time +import logging + +verbose = 1 + +class DDNS: + + def __launch(self): + pr = subprocess.Popen( + ['nsupdate', '-k', self.keyFile], + bufsize = 0, + stdin = subprocess.PIPE, + stdout = subprocess.PIPE) + + if self.nServer: + pr.stdin.write(f"server {self.nServer}\n".encode()) + + if self.zone: + pr.stdin.write(f"zone {self.zone}\n".encode()) + + return pr + + def __write(self): + diff = 0 + while True: + try: + while self.queue.qsize(): + cmd = self.queue.get() + self.nsupdate.stdin.write((cmd + "\n").encode()) + diff = 1 + logging.debug("executing command: {cmd}".format(cmd=cmd)); + + if self.nsupdate.poll(): + self.queue.put(cmd) + self.nsupdate = self.__launch() + logging.warning("Subprocess nsupdate is dead.") + + if diff and self.nsupdate.poll() == None: + diff = 0 + self.nsupdate.stdin.write(b"send\n") + + except Exception as e: + logging.warning(e) + raise Exception(e) + + time.sleep(5) + + def __init__(self, logger, keyFile, nServer, zone): + self.logger = logger + self.keyFile = keyFile + self.nServer = nServer + self.zone = zone + + self.nsupdate = self.__launch() + self.queue = Queue() + + _thread.start_new_thread(self.__write, tuple()) + + def addRecord(self, domain, rectype, value, ttl = 5): + if domain != "" and rectype != "" and value != "": + if rectype == "TXT": + value = '"%s"' % value.replace('"', '\"') + self.queue.put("update add %s %d %s %s" % (domain, ttl, rectype, value)) + + def delRecord(self, domain, rectype, value): + if domain != "": + if rectype == "TXT": + value = '"%s"' % value.replace('"', '\"') + self.queue.put("update delete %s %s %s" % (domain, rectype, value)) + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_ddns.py b/tests/test_ddns.py new file mode 100644 index 0000000..77ce66b --- /dev/null +++ b/tests/test_ddns.py @@ -0,0 +1,31 @@ +import models.ddns +import logging +import pydig +import time + +ddns = models.ddns.DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") +resolver = pydig.Resolver( + executable='/usr/bin/dig', + nameservers=[ + '172.21.21.3' + ], +) + +testdata_a = [("test.nycu-dev.me", 'A', "140.113.89.64", 5), + ("test.nycu-dev.me", 'A', "140.113.64.89", 5), + ("test2.nycu-dev.me", 'A', "140.113.69.69", 86400), +] + +def test_add_a_record(): + domains = {} + for testcase in testdata_a: + ddns.addRecord(*testcase); + if testcase[0] not in domains: + domains[testcase[0]] = set() + domains[testcase[0]].add(testcase[2]); + time.sleep(5) + for domain in domains: + assert set(resolver.query(domain, 'A')) == domains[domain] + for testcase in testdata_a: + ddns.delRecord(*testcase[:-1]) + From 17f88a02ccdc78997b3d908d25be6cb1c5ecfedb Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sun, 19 Nov 2023 17:53:48 +0000 Subject: [PATCH 04/93] add oauth --- .gitignore | 1 + config.py.sample | 19 ++++++++++++++ controllers/__init__.py | 1 + controllers/auth.py | 11 ++++++++ main.py | 28 +++++++++++++++++--- models/.ddns.py.swp | Bin 12288 -> 0 bytes models/db.py | 45 ++++++++++++++++++++++++++++++++ services/__init__.py | 2 ++ services/authService.py | 34 ++++++++++++++++++++++++ services/nctu_oauth/README.md | 5 ++++ services/nctu_oauth/__init__.py | 3 +++ services/nctu_oauth/oauth.py | 41 +++++++++++++++++++++++++++++ 12 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 .gitignore create mode 100644 config.py.sample create mode 100644 controllers/__init__.py create mode 100644 controllers/auth.py delete mode 100644 models/.ddns.py.swp create mode 100644 models/db.py create mode 100644 services/__init__.py create mode 100644 services/authService.py create mode 100644 services/nctu_oauth/README.md create mode 100644 services/nctu_oauth/__init__.py create mode 100644 services/nctu_oauth/oauth.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4acd06b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +config.py diff --git a/config.py.sample b/config.py.sample new file mode 100644 index 0000000..afe634f --- /dev/null +++ b/config.py.sample @@ -0,0 +1,19 @@ +# Oauth Parameters +NYCU_Oauth_ID = r"" +NYCU_Oauth_key = r"" +NYCU_Oauth_rURL = r"https://nycu-dev.me/oauth" + +#MySQL Parameters +MySQL_Host = r"172.21.21.2" +MySQL_User = r"nycu_me" +MySQL_Pswd = r"abc123" +MySQL_DB = r"nycu_me_dns" + +#JWT Secret Key +JWT_secretKey = r"abc123" + +#DDNS +DDNS_KeyFile = r"/etc/ddnskey.conf" +DDNS_Server = r"172.21.21.3" +DDNS_Zone = r"nycu-dev.me" + diff --git a/controllers/__init__.py b/controllers/__init__.py new file mode 100644 index 0000000..35c1920 --- /dev/null +++ b/controllers/__init__.py @@ -0,0 +1 @@ +from .auth import * diff --git a/controllers/auth.py b/controllers/auth.py new file mode 100644 index 0000000..f5de719 --- /dev/null +++ b/controllers/auth.py @@ -0,0 +1,11 @@ +from flask import Response, request +from main import app, g, nycu_oauth, authService + +@app.route("/oauth/", methods = ['GET']) +def get_token(code): + + token = nycu_oauth.get_token(code) + if token: + return {"token": authService.issue_token(nycu_oauth.get_profile(token))} + else: + return {"message": "Invalid code."}, 401 diff --git a/main.py b/main.py index 783acc5..9868e13 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,31 @@ -from flask import Flask +from flask import Flask, g, Response, request, abort +import flask_cors +import logging +from sqlalchemy import create_engine + +from config import * +from services import * + app = Flask(__name__) +flask_cors.CORS(app) + +sql_engine = create_engine( + 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( + user=MySQL_User, + pswd=MySQL_Pswd, + host=MySQL_Host, + db=MySQL_DB + ) +) + +nycu_oauth = Oauth(redirect_uri = NYCU_Oauth_rURL, + client_id = NYCU_Oauth_ID, + client_secret = NYCU_Oauth_key) + +authService = AuthService(logging, JWT_secretKey, sql_engine) @app.route('/') def hello_world(): return 'Hello, World.' -if __name__ == '__main__': - app.run(debug=True) +from controllers import * diff --git a/models/.ddns.py.swp b/models/.ddns.py.swp deleted file mode 100644 index 76c507a6aaf15712c1192b1640285b2dcb8743aa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHNzi%8x6rLayAaN2z8Y(ndWZhbv_iRuKjE*WX7>Ovx=4S$@z1_L<%KIxbyKyci zAn_kSC@6p+3K|ep2nlF`0zr`)DF|8=iU^4=d~bI5Zr465C=x})jPmWy&3p6Bd*8g7 zla=c)ys+{tf81GMX!kSr;?FOOU*Fo#x;O7;VGz^kEn7VD^7%;2N3P0*+}r&P5OzxE zP7t|6p*n-mRhu0*O*>KIi%@lZKUPjU+MC5#i~>f1+f(2Wd+gCgebgVej`5=pJb!yM zW|mRFC}0#Y3K#{90!9I&fKk9Ga7QYTrTf^Mi2SZH`jzs%Z^yg*XkJDEqkvJsC}0#Y z3K#{90!9I&fKk9GU=%P4+<^+XK4S;>GxpRy2p<3cU;X`m{Rm^%fFFQwfX{$Wfh)j= zz&pU(zzBE+Xah%qKMphY9q=viG0+EG;2GcuZ~*xG5MwuhUw|KhuYga0*MU`F2KaM^ zv5$Z&fCQF-!@&0k8T%6023`YZfd_%V4ls5V_yYJGco(<~oB_JPL%_}Z82cUg4fq=P z0C*qp0S}l1uHVbpPrzm1NuUeN05_n|Rp33~X@K-v223*w7zK<1MuGpg0=(9g2nWu& zT;#$@^USvL)ORz%@$)u;*Q_f3S$vk;eiFGs+~!hv*(eole%=jpF*nC*bh@j@d6&1W zS!J~_RS!5)q@n8xyJfZ7ymi)UHD?BcikWx2+**HW-C7z`!&ofOkHJaIxx~g)6yhFPyS}gUEuN~8b(V#@SepqD@qeE>cVmAnlomse6btVV2l@O9 zaoQZYVY`mc)l}+b8&bHwqcT@!y;y9O16pRM*V~doh7!o~G!!;!fKm3;Ceng(l?uzA z$fS+2DvPXJIQM#?o5$V;{FtVwSS}{9pn)!@9-}U`f@Dp|^FqQF7d@tJm3dQ)mV;2h zT=tsgFc}Up8Rnv=tyE6EUJwUauUA%6VQp6=?P{^Mj;O8|t)2yua8xKnY9FuVLMCd` z4V2&~FL)x&f+V)ZM3=&W>u7>_=xn(%#(SrW&~qmg+*Mpm4hgZaimLu@$c`=bgV@m< zX!k88V*hMx?T&>A{9rJENDHjaMtwSuWTRqivZqNH+H-V-Phr!V!O=`ck7 zi+(<|Epb73d4@E(mqZcNTjJYT>5}CP5*fLfjSpQ~dhQ8Ur>XkohAd7sHp8lKGtu%# zxJ4W7h}gr=m+8<}to1i}mZBA63`JHu!IkUQ1|2WGV&Qg91s6fu-0)0}CIiKkb(znTHZRh;4QsCO^no-ewGhxQIcnL4XJUhJEvDppKNZ8-bqZ^<<@Sn_45Jk0o)dv zhBK||{LmNg{3f1SnDT0Tb>f}Ug<6j#NkweyRf^3iZ|ZV)n1^4)D}Lg{sWp^8uH$}! q8?MV284IE`ky&-2RUhewWCNK*To(;5y;kXKqVg%fXsVBsXxKjigQyk& diff --git a/models/db.py b/models/db.py new file mode 100644 index 0000000..e179d35 --- /dev/null +++ b/models/db.py @@ -0,0 +1,45 @@ +from sqlalchemy import create_engine, Column, String, Integer, DateTime, ForeignKey, Text, CHAR, BOOLEAN, UniqueConstraint +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +Base = declarative_base() + +class User(Base): + __tablename__ = 'users' + + id = Column(String(16), primary_key=True) + name = Column(String(256), nullable=False) + username = Column(String(256), nullable=False) + password = Column(String(100), nullable=False, default='') + status = Column(String(16), nullable=False) + email = Column(String(256), nullable=False) + limit = Column(Integer, nullable=False, default=2) + isAdmin = Column(Integer, nullable=False, default=0) + + # Relationships + domains = relationship("Domain", backref="user") + +class Domain(Base): + __tablename__ = 'domains' + + id = Column(Integer, primary_key=True, autoincrement=True) + userId = Column(String(16), ForeignKey('users.id'), nullable=False) + domain = Column(Text) + regDate = Column(DateTime) + expDate = Column(DateTime) + status = Column(BOOLEAN, default=True) + + # Relationships + records = relationship("Record", backref="domain") + +class Record(Base): + __tablename__ = 'records' + + id = Column(Integer, primary_key=True, autoincrement=True) + domain_id = Column(Integer, ForeignKey('domains.id'), nullable=False) + type = Column(CHAR(16), nullable=False) + value = Column(String(256)) + ttl = Column(Integer, nullable=False) + regDate = Column(DateTime) + expDate = Column(DateTime) + status = Column(BOOLEAN, default=True) diff --git a/services/__init__.py b/services/__init__.py new file mode 100644 index 0000000..a256a59 --- /dev/null +++ b/services/__init__.py @@ -0,0 +1,2 @@ +from .authService import AuthService +from .nctu_oauth import * diff --git a/services/authService.py b/services/authService.py new file mode 100644 index 0000000..ffbd37e --- /dev/null +++ b/services/authService.py @@ -0,0 +1,34 @@ +from sqlalchemy.orm import sessionmaker +from datetime import timezone, datetime +from models import db +import jwt + +class AuthService: + def __init__(self, logger, jwt_secret, sql_engine): + self.logger = logger + self.jwt_secret = jwt_secret + self.sql_engine = sql_engine + + def issue_token(self, profile): + now = int(datetime.now(tz=timezone.utc).timestamp()) + token = profile + token['iss'] = 'dns.nycu.me' + token['exp'] = (now) + 3600 + token['iat'] = token['nbf'] = now + token['uid'] = token['username'] + + Session = sessionmaker(bind=self.sql_engine) + session = Session() + user = session.query(db.User).filter_by(id=token['uid']).all() + + if len(user): + user = user[0] + if user.email != token['email']: + user.email = token['email'] + session.commit() + else: + user = db.User(id=token['uid'], name='', username='', password='', status='active', email=token['email']) + session.add(user) + session.commit() + token = jwt.encode(token, self.jwt_secret, algorithm="HS256") + return token diff --git a/services/nctu_oauth/README.md b/services/nctu_oauth/README.md new file mode 100644 index 0000000..52ba3b5 --- /dev/null +++ b/services/nctu_oauth/README.md @@ -0,0 +1,5 @@ +# NCTU oauth + +CREDIT TO steven5538. + +You can find the source repo [here](https://github.com/steven5538/NCTU-Oauth). diff --git a/services/nctu_oauth/__init__.py b/services/nctu_oauth/__init__.py new file mode 100644 index 0000000..465ec8f --- /dev/null +++ b/services/nctu_oauth/__init__.py @@ -0,0 +1,3 @@ +#-*- encoding: UTF-8 -*- + +from .oauth import Oauth \ No newline at end of file diff --git a/services/nctu_oauth/oauth.py b/services/nctu_oauth/oauth.py new file mode 100644 index 0000000..1914838 --- /dev/null +++ b/services/nctu_oauth/oauth.py @@ -0,0 +1,41 @@ +#-*- encoding: UTF-8 -*- + +import requests + +OAUTH_URL = 'https://id.nycu.edu.tw' + +class Oauth(object): + def __init__(self, redirect_uri, client_id, client_secret): + self.grant_type = 'authorization_code' + self.client_id = client_id + self.client_secret = client_secret + self.redirect_uri = redirect_uri + + def get_token(self, code): + + get_token_url = OAUTH_URL + '/o/token/' + data = { + 'grant_type': 'authorization_code', + 'code': code, + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'redirect_uri': self.redirect_uri + } + result = requests.post(get_token_url, data=data) + access_token = result.json().get('access_token', None) + + if access_token: + return access_token + + return False + + def get_profile(self, token): + + headers = { + 'Authorization': 'Bearer ' + token + } + get_profile_url = OAUTH_URL + '/api/profile/' + + data = requests.get(get_profile_url, headers=headers).json() + + return data From 181f30c6836c183ae4773fb62640d20ede8a0c79 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 20 Nov 2023 05:09:05 +0000 Subject: [PATCH 05/93] add authenticate function --- .gitignore | 1 + controllers/auth.py | 20 +++++++++++++++++++- services/authService.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4acd06b..365b5da 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ config.py +*.swp diff --git a/controllers/auth.py b/controllers/auth.py index f5de719..cb6fbfd 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -1,11 +1,29 @@ from flask import Response, request from main import app, g, nycu_oauth, authService + +@app.before_request +def before_request(): + + g.user = authService.authenticate_token(request.headers.get('Authorization')) + @app.route("/oauth/", methods = ['GET']) def get_token(code): - + token = nycu_oauth.get_token(code) + if token: return {"token": authService.issue_token(nycu_oauth.get_profile(token))} else: return {"message": "Invalid code."}, 401 + +@app.route("/whoami/", methods = ['GET']) +def whoami(): + + if g.user: + data = {} + data['uid'] = g.user['uid'] + data['email'] = g.user['email'] + return data + + return {"message": "Unauth."}, 401 diff --git a/services/authService.py b/services/authService.py index ffbd37e..c349ee6 100644 --- a/services/authService.py +++ b/services/authService.py @@ -3,6 +3,16 @@ from models import db import jwt +class UnauthorizedError(Exception): + def __init__(self, msg): + self.msg = str(msg) + + def __str__(self): + return self.msg + + def __repr__(self): + return "Unauthorized: " + self.msg + class AuthService: def __init__(self, logger, jwt_secret, sql_engine): self.logger = logger @@ -32,3 +42,24 @@ def issue_token(self, profile): session.commit() token = jwt.encode(token, self.jwt_secret, algorithm="HS256") return token + + def authenticate_token(self, payload): + try: + if not payload: + raise UnauthorizedError("not logged") + payload = payload.split(' ') + if len(payload) != 2: + raise UnauthorizedError("invalid payload") + tokenType, token = payload + if tokenType != 'Bearer': + raise UnauthorizedError("invalid payload") + try: + payload = jwt.decode(token, self.jwt_secret, algorithms=["HS256"]) + except Exception as e: + self.logger.debug(e.__str__()) + return None + return payload + except UnauthorizedError as e: + self.logger.debug(e.__str__()) + return None + From c15de5b7ab18361e98ce942afc272eb2e9274760 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 20 Nov 2023 09:10:43 +0000 Subject: [PATCH 06/93] move orm operation to models layer and add unittest for login. --- main.py | 4 +++- models/__init__.py | 4 ++++ models/db.py | 2 +- models/users.py | 31 +++++++++++++++++++++++++++++++ services/authService.py | 20 +++++++------------- tests/test_ddns.py | 8 ++++---- tests/test_issue_token.py | 26 ++++++++++++++++++++++++++ 7 files changed, 76 insertions(+), 19 deletions(-) create mode 100644 models/__init__.py create mode 100644 models/users.py create mode 100644 tests/test_issue_token.py diff --git a/main.py b/main.py index 9868e13..dc80d6f 100644 --- a/main.py +++ b/main.py @@ -4,6 +4,7 @@ from sqlalchemy import create_engine from config import * +from models import users from services import * app = Flask(__name__) @@ -18,11 +19,12 @@ ) ) +users = users.Users(sql_engine) nycu_oauth = Oauth(redirect_uri = NYCU_Oauth_rURL, client_id = NYCU_Oauth_ID, client_secret = NYCU_Oauth_key) -authService = AuthService(logging, JWT_secretKey, sql_engine) +authService = AuthService(logging, JWT_secretKey, users) @app.route('/') def hello_world(): diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..3335c29 --- /dev/null +++ b/models/__init__.py @@ -0,0 +1,4 @@ +from .db import * +from .ddns import * +from .users import * + diff --git a/models/db.py b/models/db.py index e179d35..f6b5f85 100644 --- a/models/db.py +++ b/models/db.py @@ -1,5 +1,5 @@ from sqlalchemy import create_engine, Column, String, Integer, DateTime, ForeignKey, Text, CHAR, BOOLEAN, UniqueConstraint -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import declarative_base from sqlalchemy.orm import relationship Base = declarative_base() diff --git a/models/users.py b/models/users.py new file mode 100644 index 0000000..92774c1 --- /dev/null +++ b/models/users.py @@ -0,0 +1,31 @@ +from sqlalchemy.orm import sessionmaker +from . import db + +class Users: + def __init__(self, sql_engine): + Session = sessionmaker(bind=sql_engine) + self.session = Session() + + def query(self, uid): + user = self.session.query(db.User).filter_by(id=uid).all() + if len(user): + return user[0] + return None + + def add(self, uid, name, username, password, status, email): + user = db.User(id=uid, + name=name, + username=username, + password=password, + status=status, + email=email) + self.session.add(user) + self.session.commit() + + def update_email(self, uid, email): + user = self.query(uid) + user.email = email + self.session.commit() + + def __del__(self): + self.session.close() diff --git a/services/authService.py b/services/authService.py index c349ee6..b45a905 100644 --- a/services/authService.py +++ b/services/authService.py @@ -1,6 +1,5 @@ from sqlalchemy.orm import sessionmaker from datetime import timezone, datetime -from models import db import jwt class UnauthorizedError(Exception): @@ -14,10 +13,10 @@ def __repr__(self): return "Unauthorized: " + self.msg class AuthService: - def __init__(self, logger, jwt_secret, sql_engine): + def __init__(self, logger, jwt_secret, users): self.logger = logger self.jwt_secret = jwt_secret - self.sql_engine = sql_engine + self.users = users def issue_token(self, profile): now = int(datetime.now(tz=timezone.utc).timestamp()) @@ -27,19 +26,14 @@ def issue_token(self, profile): token['iat'] = token['nbf'] = now token['uid'] = token['username'] - Session = sessionmaker(bind=self.sql_engine) - session = Session() - user = session.query(db.User).filter_by(id=token['uid']).all() + user = self.users.query(token['uid']) - if len(user): - user = user[0] + if user: if user.email != token['email']: - user.email = token['email'] - session.commit() + self.users.update(token['uid'], token['email']) else: - user = db.User(id=token['uid'], name='', username='', password='', status='active', email=token['email']) - session.add(user) - session.commit() + self.users.add(uid=token['uid'], name='', username='', password='', status='active', email=token['email']) + token = jwt.encode(token, self.jwt_secret, algorithm="HS256") return token diff --git a/tests/test_ddns.py b/tests/test_ddns.py index 77ce66b..a9dfce2 100644 --- a/tests/test_ddns.py +++ b/tests/test_ddns.py @@ -11,14 +11,14 @@ ], ) -testdata_a = [("test.nycu-dev.me", 'A', "140.113.89.64", 5), +testdata_A = [("test.nycu-dev.me", 'A', "140.113.89.64", 5), ("test.nycu-dev.me", 'A', "140.113.64.89", 5), ("test2.nycu-dev.me", 'A', "140.113.69.69", 86400), ] -def test_add_a_record(): +def test_add_A_record(): domains = {} - for testcase in testdata_a: + for testcase in testdata_A: ddns.addRecord(*testcase); if testcase[0] not in domains: domains[testcase[0]] = set() @@ -26,6 +26,6 @@ def test_add_a_record(): time.sleep(5) for domain in domains: assert set(resolver.query(domain, 'A')) == domains[domain] - for testcase in testdata_a: + for testcase in testdata_A: ddns.delRecord(*testcase[:-1]) diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py new file mode 100644 index 0000000..904efa0 --- /dev/null +++ b/tests/test_issue_token.py @@ -0,0 +1,26 @@ +from sqlalchemy import create_engine +import logging + +from services.authService import AuthService +from models import users +from config import * + +sql_engine = create_engine( + 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( + user=MySQL_User, + pswd=MySQL_Pswd, + host=MySQL_Host, + db=MySQL_DB + ) +) +users = users.Users(sql_engine) +authService = AuthService(logging, JWT_secretKey, users) + +testdata = [{"email":"lin.cs09@nycu.edu.tw","username":"109550028"}] + +def test_issue_token(): + for testcase in testdata: + token = "Bearer " + authService.issue_token(testcase) + assert authService.authenticate_token(token) + assert authService.authenticate_token(token + 'a') == None # modified token + From aa91768391bae107ed98c5cf3c0b4c151d56c005 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 20 Nov 2023 10:05:03 +0000 Subject: [PATCH 07/93] use sqlite to test in order to reduce database dependency on testing. --- tests/test_issue_token.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py index 904efa0..0b348cd 100644 --- a/tests/test_issue_token.py +++ b/tests/test_issue_token.py @@ -1,26 +1,28 @@ +from unittest.mock import create_autospec from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.engine.base import Engine import logging from services.authService import AuthService -from models import users +from models import users, db from config import * -sql_engine = create_engine( - 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( - user=MySQL_User, - pswd=MySQL_Pswd, - host=MySQL_Host, - db=MySQL_DB - ) -) +sql_engine = create_engine('sqlite:///:memory:') +db.Base.metadata.create_all(sql_engine) +Session = sessionmaker(bind=sql_engine) +session = Session() + users = users.Users(sql_engine) authService = AuthService(logging, JWT_secretKey, users) -testdata = [{"email":"lin.cs09@nycu.edu.tw","username":"109550028"}] +testdata = [{'email':"lin.cs09@nycu.edu.tw",'username':"109550028"}] def test_issue_token(): for testcase in testdata: token = "Bearer " + authService.issue_token(testcase) - assert authService.authenticate_token(token) - assert authService.authenticate_token(token + 'a') == None # modified token - + assert authService.authenticate_token(token) != None + # test modified token + assert authService.authenticate_token(token + 'a') == None + # test if data is written + assert len(session.query(db.User).filter_by(id=testcase['username']).all()) From 1404dc9ac4c28df79c079120e85dd398a8eb8a79 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 20 Nov 2023 17:15:59 +0000 Subject: [PATCH 08/93] add DNSService and its dependency --- controllers/auth.py | 4 +- controllers/domains.py | 16 +++++++ main.py | 28 +++++------ models/__init__.py | 3 +- models/ddns.py | 4 +- models/domains.py | 50 +++++++++++++++++++ models/records.py | 37 ++++++++++++++ models/users.py | 2 +- services/__init__.py | 3 +- services/authService.py | 2 +- services/dnsService.py | 90 +++++++++++++++++++++++++++++++++++ tests/test_ddns.py | 8 ++-- tests/test_domain_register.py | 46 ++++++++++++++++++ tests/test_issue_token.py | 21 ++++---- 14 files changed, 279 insertions(+), 35 deletions(-) create mode 100644 controllers/domains.py create mode 100644 models/domains.py create mode 100644 models/records.py create mode 100644 services/dnsService.py create mode 100644 tests/test_domain_register.py diff --git a/controllers/auth.py b/controllers/auth.py index cb6fbfd..c7f6cf9 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -1,5 +1,5 @@ -from flask import Response, request -from main import app, g, nycu_oauth, authService +from flask import g +from main import app, nycu_oauth, authService @app.before_request diff --git a/controllers/domains.py b/controllers/domains.py new file mode 100644 index 0000000..7184ea6 --- /dev/null +++ b/controllers/domains.py @@ -0,0 +1,16 @@ +from flask import Response, request +from main import app, g, authService + + +@app.route("/domains/", methods=['POST']) +def applyDomain(domain): + + if not g.user: + return {"message": "Unauth."}, 401 + + user = users.getUser(g.user['uid']) + domainStruct = domain.lower().strip('/').split('/') + domainName = '.'.join(reversed(domainStruct)) + domain = dns.getDomain(domainName) + + diff --git a/main.py b/main.py index dc80d6f..9664df3 100644 --- a/main.py +++ b/main.py @@ -1,33 +1,33 @@ -from flask import Flask, g, Response, request, abort +from flask import Flask import flask_cors import logging from sqlalchemy import create_engine -from config import * -from models import users -from services import * +import config +from models import Users +from services import AuthService, DNSService, Oauth app = Flask(__name__) flask_cors.CORS(app) sql_engine = create_engine( 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( - user=MySQL_User, - pswd=MySQL_Pswd, - host=MySQL_Host, - db=MySQL_DB + user=config.MYSQL_USER, + pswd=config.MYSQL_PSWD, + host=config.MYSQL_HOST, + db=config.MYSQL_DB ) ) -users = users.Users(sql_engine) -nycu_oauth = Oauth(redirect_uri = NYCU_Oauth_rURL, - client_id = NYCU_Oauth_ID, - client_secret = NYCU_Oauth_key) +users = Users(sql_engine) +nycu_oauth = Oauth(redirect_uri = config.NYCU_OAUTH_RURL, + client_id = config.NYCU_OAUTH_ID, + client_secret = config.NYCU_OAUTH_KEY) -authService = AuthService(logging, JWT_secretKey, users) +authService = AuthService(logging, config.JWT_SECRET, users) @app.route('/') def hello_world(): return 'Hello, World.' -from controllers import * +from controllers import auth diff --git a/models/__init__.py b/models/__init__.py index 3335c29..f1d3393 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -1,4 +1,5 @@ from .db import * from .ddns import * from .users import * - +from .domains import * +from .records import * diff --git a/models/ddns.py b/models/ddns.py index 7e22542..491b5e8 100644 --- a/models/ddns.py +++ b/models/ddns.py @@ -59,13 +59,13 @@ def __init__(self, logger, keyFile, nServer, zone): _thread.start_new_thread(self.__write, tuple()) - def addRecord(self, domain, rectype, value, ttl = 5): + def add_record(self, domain, rectype, value, ttl = 5): if domain != "" and rectype != "" and value != "": if rectype == "TXT": value = '"%s"' % value.replace('"', '\"') self.queue.put("update add %s %d %s %s" % (domain, ttl, rectype, value)) - def delRecord(self, domain, rectype, value): + def del_record(self, domain, rectype, value): if domain != "": if rectype == "TXT": value = '"%s"' % value.replace('"', '\"') diff --git a/models/domains.py b/models/domains.py new file mode 100644 index 0000000..862651c --- /dev/null +++ b/models/domains.py @@ -0,0 +1,50 @@ +from sqlalchemy.orm import sessionmaker +from datetime import datetime, timedelta + +from . import db + + +class Domains: + def __init__(self, sql_engine): + Session = sessionmaker(bind=sql_engine) + self.session = Session() + + def get_domain(self, domain): + domain = self.session.query(db.Domain).filter_by(domain=domain, status=1).all() + if domain: + return domain[0] + return None + + def get_domain_by_id(self, domain_id): + domain = self.session.query(db.Domain).filter_by(id=domain_id, status=1).all() + if domain: + return domain[0] + return None + + def list_by_user(self, user): + domains = self.session.query(db.Domain).filter_by(userId=user, status=1).all() + return domains + + def register(self, domain, user): + now = datetime.now() + regDate = now + expDate = (now + timedelta(days=30)) + domain = db.Domain(userId = user, domain = domain, regDate = regDate, expDate = expDate, status = 1) + self.session.add(domain) + self.session.commit() + + def renew(self, domain): + domain = self.get_domain(domain) + if domain != None: + now = datetime.now() + expDate = (now + timedelta(days=30)) + domain.expDate = expDate + self.session.commit() + + def release(self, domain): + domain = self.get_domain(domain) + if domain != None: + expDate = datetime.now() + domain.expDate = expDate + domain.status = 0 + self.session.commit() diff --git a/models/records.py b/models/records.py new file mode 100644 index 0000000..89d18e7 --- /dev/null +++ b/models/records.py @@ -0,0 +1,37 @@ +from sqlalchemy.orm import sessionmaker +from datetime import datetime + +from . import db + + +class Records: + def __init__(self, sql_engine): + Session = sessionmaker(bind=sql_engine) + self.session = Session() + + def get_record(self, record_id): + return self.session.query(db.Record).filter_by(id=record_id, status=1).first() + + def get_records(self, domain_id): + return self.session.query(db.Record).filter_by(domain_id=domain_id, status=1).all() + + def add_record(self, domain_id, record_type, value, ttl): + now = datetime.now() + regDate = now + record = db.Record(domain_id=domain_id, + type=record_type, + value=value, + ttl=ttl, + regDate=regDate, + status=1) + self.session.add(record) + self.session.commit() + + def del_record(self, record_id): + records = self.session.query(db.Record).filter_by(id=record_id).all() + now = datetime.now() + expDate = now + for record in records: + record.status = 0 + record.expDate = now + self.session.commit() diff --git a/models/users.py b/models/users.py index 92774c1..092a3c7 100644 --- a/models/users.py +++ b/models/users.py @@ -1,6 +1,6 @@ from sqlalchemy.orm import sessionmaker from . import db - +import logging class Users: def __init__(self, sql_engine): Session = sessionmaker(bind=sql_engine) diff --git a/services/__init__.py b/services/__init__.py index a256a59..3c08951 100644 --- a/services/__init__.py +++ b/services/__init__.py @@ -1,2 +1,3 @@ -from .authService import AuthService +from .authService import * +from .dnsService import * from .nctu_oauth import * diff --git a/services/authService.py b/services/authService.py index b45a905..2a40575 100644 --- a/services/authService.py +++ b/services/authService.py @@ -30,7 +30,7 @@ def issue_token(self, profile): if user: if user.email != token['email']: - self.users.update(token['uid'], token['email']) + self.users.update_email(token['uid'], token['email']) else: self.users.add(uid=token['uid'], name='', username='', password='', status='active', email=token['email']) diff --git a/services/dnsService.py b/services/dnsService.py new file mode 100644 index 0000000..938e666 --- /dev/null +++ b/services/dnsService.py @@ -0,0 +1,90 @@ +import time +from enum import Enum + + +class DNSErrors(Enum): + NXDomain = "Non-ExistentDomain" + NotAllowedRecordType = "NotAllowedRecordType" + DuplicatedRecord = "DuplicatedRecord" + NotAllowedOperation = "NotAllowedOperation" + +class DNSError(Exception): + def __init__(self, typ, msg = ""): + self.typ = str(typ) + self.msg = str(msg) + + def __str__(self): + return self.msg + + def __repr__(self): + return "%s: %s" % (self.typ, self.msg) + +class DNSService(): + def __init__(self, logger, users, domains, records, ddns): + self.logger = logger + self.users = users + self.domains = domains + self.records = records + self.ddns = ddns + + def get_domain(self, domain_name): + domain = self.domains.get_domain(domain_name) + if not domain: + return None + + domain_info = {} + domain_info['id'] = domain.id + domain_info['regDate'] = domain.regDate + domain_info['expDate'] = domain.expDate + domain_info['domain'] = domain_name + domain_info['records'] = [] + records = self.records.get_records(domain.id) + for record in records: + domain_info['records'].append((record.id, + record.type, + record.value, + record.ttl)) + return domain_info + + def register_domain(self, uid, domain_name): + domain = self.domains.get_domain(domain_name) + if domain: + raise DNSError(DNSErrors.NotAllowedOperation, "This domain have been registered.") + self.domains.register(domain_name, uid) + + def renew_domain(self, domain_name): + domain = self.domains.get_domain(domain_name) + if not domain: + raise DNSError(DNSErrors.NotAllowedOperation, "This domain is not registered.") + self.domains.renew(domain_name) + + def release_domain(self, domain_name): + domain = self.domains.get_domain(domain_name) + if not domain: + raise DNSError(DNSErrors.NotAllowedOperation, "This domain is not registered.") + records = self.records.get_records(domain.id) + for record in records: + self.del_record(record.id) + self.domains.release(domain_name) + + def add_record(self, domain_name, type_, value, ttl): + domain = self.domains.get_domain(domain_name) + if not domain: + raise DNSError(DNSErrors.NotAllowedOperation, "This domain is not registered.") + + records = self.records.get_records(domain.id) + for record in records: + if type_ == record.type and value == record.value: + raise DNSError(DNSErrors.DuplicatedRecord, "You have created same record.") + + self.records.add_record(domain.id, type_, value, ttl) + self.ddns.add_record(domain_name, type_, value, ttl) + + def del_record(self, record_id): + record = self.records.get_record(record_id) + if not record: + raise DNSError(DNSErrors.NotAllowedOperation, "This record does not exist.") + + domain = self.domains.get_domain_by_id(record.domain_id) + self.records.del_record(record_id) + self.ddns.del_record(domain.domain, record.type, record.value) diff --git a/tests/test_ddns.py b/tests/test_ddns.py index a9dfce2..5eadea4 100644 --- a/tests/test_ddns.py +++ b/tests/test_ddns.py @@ -19,7 +19,7 @@ def test_add_A_record(): domains = {} for testcase in testdata_A: - ddns.addRecord(*testcase); + ddns.add_record(*testcase); if testcase[0] not in domains: domains[testcase[0]] = set() domains[testcase[0]].add(testcase[2]); @@ -27,5 +27,7 @@ def test_add_A_record(): for domain in domains: assert set(resolver.query(domain, 'A')) == domains[domain] for testcase in testdata_A: - ddns.delRecord(*testcase[:-1]) - + ddns.del_record(*testcase[:-1]) + time.sleep(5) + for domain in domains: + assert resolver.query(domain, 'A') == [] diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py new file mode 100644 index 0000000..d796014 --- /dev/null +++ b/tests/test_domain_register.py @@ -0,0 +1,46 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +import logging +import pydig +import time + +from models import Domains, Records, Users, db, DDNS +from services import AuthService, DNSService +import config + + +ddns = DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") + +resolver = pydig.Resolver( + executable='/usr/bin/dig', + nameservers=[ + '172.21.21.3' + ], +) + +sql_engine = create_engine('sqlite:///:memory:') +db.Base.metadata.create_all(sql_engine) +Session = sessionmaker(bind=sql_engine) +session = Session() + +users = Users(sql_engine) +domains = Domains(sql_engine) +records = Records(sql_engine) + +authService = AuthService(logging, config.JWT_SECRET, users) +dnsService = DNSService(logging, users, domains, records, ddns) + +testdata = [("test.nycu-dev.me", 'A', "140.113.89.64", 5), + ("test.nycu-dev.me", 'A', "140.113.64.89", 5)] +answer = {"140.113.89.64", "140.113.64.89"} + +def test_domain_register(): + dnsService.register_domain("109550028", "test.nycu-dev.me") + for testcase in testdata: + dnsService.add_record(*testcase) + time.sleep(10) + assert set(resolver.query("test.nycu-dev.me", 'A')) == answer + dnsService.release_domain("test.nycu-dev.me") + dnsService.register_domain("109550028", "test") + time.sleep(10) + assert set(resolver.query("test.nycu-dev.me", 'A')) == set() diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py index 0b348cd..7e76fe5 100644 --- a/tests/test_issue_token.py +++ b/tests/test_issue_token.py @@ -1,12 +1,10 @@ -from unittest.mock import create_autospec from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from sqlalchemy.engine.base import Engine import logging from services.authService import AuthService from models import users, db -from config import * +import config sql_engine = create_engine('sqlite:///:memory:') db.Base.metadata.create_all(sql_engine) @@ -14,15 +12,18 @@ session = Session() users = users.Users(sql_engine) -authService = AuthService(logging, JWT_secretKey, users) +authService = AuthService(logging, config.JWT_SECRET, users) -testdata = [{'email':"lin.cs09@nycu.edu.tw",'username':"109550028"}] +testdata = [{'email':"lin.cs09@nycu.edu.tw",'username':"109550028"}, + {'email':"lin.cs09@nctu.edu.tw",'username':"109550028"}] def test_issue_token(): - for testcase in testdata: - token = "Bearer " + authService.issue_token(testcase) - assert authService.authenticate_token(token) != None + for testcase in testdata: + token = "Bearer " + authService.issue_token(testcase) + assert authService.authenticate_token(token) != None # test modified token - assert authService.authenticate_token(token + 'a') == None + assert authService.authenticate_token(token + 'a') == None # test if data is written - assert len(session.query(db.User).filter_by(id=testcase['username']).all()) + session.expire_all() # flush orm cache + data = session.query(db.User).filter_by(id=testcase['username']).all() + assert len(data) and data[0].email == testcase['email'] From 199b523fd1b5b8ba5767c40ba1aa7689bb8eb00e Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 20 Nov 2023 17:55:12 +0000 Subject: [PATCH 09/93] improve lint rating --- controllers/auth.py | 7 +---- main.py | 8 ++---- models/domains.py | 6 ++++- models/records.py | 5 ++-- services/authService.py | 11 +++++--- services/dnsService.py | 19 +++++++------ services/nctu_oauth/oauth.py | 52 ++++++++++++++++++------------------ 7 files changed, 53 insertions(+), 55 deletions(-) diff --git a/controllers/auth.py b/controllers/auth.py index c7f6cf9..ac84b10 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -1,17 +1,14 @@ -from flask import g +from flask import request, g from main import app, nycu_oauth, authService @app.before_request def before_request(): - g.user = authService.authenticate_token(request.headers.get('Authorization')) @app.route("/oauth/", methods = ['GET']) def get_token(code): - token = nycu_oauth.get_token(code) - if token: return {"token": authService.issue_token(nycu_oauth.get_profile(token))} else: @@ -19,11 +16,9 @@ def get_token(code): @app.route("/whoami/", methods = ['GET']) def whoami(): - if g.user: data = {} data['uid'] = g.user['uid'] data['email'] = g.user['email'] return data - return {"message": "Unauth."}, 401 diff --git a/main.py b/main.py index 9664df3..b1a602c 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,6 @@ +import logging from flask import Flask import flask_cors -import logging from sqlalchemy import create_engine import config @@ -11,7 +11,7 @@ flask_cors.CORS(app) sql_engine = create_engine( - 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( + f'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( user=config.MYSQL_USER, pswd=config.MYSQL_PSWD, host=config.MYSQL_HOST, @@ -26,8 +26,4 @@ authService = AuthService(logging, config.JWT_SECRET, users) -@app.route('/') -def hello_world(): - return 'Hello, World.' - from controllers import auth diff --git a/models/domains.py b/models/domains.py index 862651c..f2ad882 100644 --- a/models/domains.py +++ b/models/domains.py @@ -29,7 +29,11 @@ def register(self, domain, user): now = datetime.now() regDate = now expDate = (now + timedelta(days=30)) - domain = db.Domain(userId = user, domain = domain, regDate = regDate, expDate = expDate, status = 1) + domain = db.Domain(userId = user, + domain = domain, + regDate = regDate, + expDate = expDate, + status = 1) self.session.add(domain) self.session.commit() diff --git a/models/records.py b/models/records.py index 89d18e7..cfce1c1 100644 --- a/models/records.py +++ b/models/records.py @@ -11,10 +11,10 @@ def __init__(self, sql_engine): def get_record(self, record_id): return self.session.query(db.Record).filter_by(id=record_id, status=1).first() - + def get_records(self, domain_id): return self.session.query(db.Record).filter_by(domain_id=domain_id, status=1).all() - + def add_record(self, domain_id, record_type, value, ttl): now = datetime.now() regDate = now @@ -30,7 +30,6 @@ def add_record(self, domain_id, record_type, value, ttl): def del_record(self, record_id): records = self.session.query(db.Record).filter_by(id=record_id).all() now = datetime.now() - expDate = now for record in records: record.status = 0 record.expDate = now diff --git a/services/authService.py b/services/authService.py index 2a40575..d2ef518 100644 --- a/services/authService.py +++ b/services/authService.py @@ -25,18 +25,23 @@ def issue_token(self, profile): token['exp'] = (now) + 3600 token['iat'] = token['nbf'] = now token['uid'] = token['username'] - + user = self.users.query(token['uid']) if user: if user.email != token['email']: self.users.update_email(token['uid'], token['email']) else: - self.users.add(uid=token['uid'], name='', username='', password='', status='active', email=token['email']) + self.users.add(uid=token['uid'], + name='', + username='', + password='', + status='active', + email=token['email']) token = jwt.encode(token, self.jwt_secret, algorithm="HS256") return token - + def authenticate_token(self, payload): try: if not payload: diff --git a/services/dnsService.py b/services/dnsService.py index 938e666..a0cd971 100644 --- a/services/dnsService.py +++ b/services/dnsService.py @@ -3,10 +3,9 @@ class DNSErrors(Enum): - NXDomain = "Non-ExistentDomain" - NotAllowedRecordType = "NotAllowedRecordType" - DuplicatedRecord = "DuplicatedRecord" - NotAllowedOperation = "NotAllowedOperation" + NXDOMAIN = "Non-ExistentDomain" + DUPLICATED = "DuplicatedRecord" + UNALLOWED = "NotAllowedOperation" class DNSError(Exception): def __init__(self, typ, msg = ""): @@ -49,19 +48,19 @@ def get_domain(self, domain_name): def register_domain(self, uid, domain_name): domain = self.domains.get_domain(domain_name) if domain: - raise DNSError(DNSErrors.NotAllowedOperation, "This domain have been registered.") + raise DNSError(DNSErrors.UNALLOWED, "This domain have been registered.") self.domains.register(domain_name, uid) def renew_domain(self, domain_name): domain = self.domains.get_domain(domain_name) if not domain: - raise DNSError(DNSErrors.NotAllowedOperation, "This domain is not registered.") + raise DNSError(DNSErrors.NXDOMAIN, "This domain is not registered.") self.domains.renew(domain_name) def release_domain(self, domain_name): domain = self.domains.get_domain(domain_name) if not domain: - raise DNSError(DNSErrors.NotAllowedOperation, "This domain is not registered.") + raise DNSError(DNSErrors.NXDOMAIN, "This domain is not registered.") records = self.records.get_records(domain.id) for record in records: self.del_record(record.id) @@ -70,12 +69,12 @@ def release_domain(self, domain_name): def add_record(self, domain_name, type_, value, ttl): domain = self.domains.get_domain(domain_name) if not domain: - raise DNSError(DNSErrors.NotAllowedOperation, "This domain is not registered.") + raise DNSError(DNSErrors.NXDOMAIN, "This domain is not registered.") records = self.records.get_records(domain.id) for record in records: if type_ == record.type and value == record.value: - raise DNSError(DNSErrors.DuplicatedRecord, "You have created same record.") + raise DNSError(DNSErrors.DUPLICATED, "You have created same record.") self.records.add_record(domain.id, type_, value, ttl) self.ddns.add_record(domain_name, type_, value, ttl) @@ -83,7 +82,7 @@ def add_record(self, domain_name, type_, value, ttl): def del_record(self, record_id): record = self.records.get_record(record_id) if not record: - raise DNSError(DNSErrors.NotAllowedOperation, "This record does not exist.") + raise DNSError(DNSErrors.UNALLOWED, "This record does not exist.") domain = self.domains.get_domain_by_id(record.domain_id) self.records.del_record(record_id) diff --git a/services/nctu_oauth/oauth.py b/services/nctu_oauth/oauth.py index 1914838..42e9a0b 100644 --- a/services/nctu_oauth/oauth.py +++ b/services/nctu_oauth/oauth.py @@ -5,37 +5,37 @@ OAUTH_URL = 'https://id.nycu.edu.tw' class Oauth(object): - def __init__(self, redirect_uri, client_id, client_secret): - self.grant_type = 'authorization_code' - self.client_id = client_id - self.client_secret = client_secret - self.redirect_uri = redirect_uri + def __init__(self, redirect_uri, client_id, client_secret): + self.grant_type = 'authorization_code' + self.client_id = client_id + self.client_secret = client_secret + self.redirect_uri = redirect_uri - def get_token(self, code): + def get_token(self, code): - get_token_url = OAUTH_URL + '/o/token/' - data = { - 'grant_type': 'authorization_code', - 'code': code, - 'client_id': self.client_id, - 'client_secret': self.client_secret, - 'redirect_uri': self.redirect_uri - } - result = requests.post(get_token_url, data=data) - access_token = result.json().get('access_token', None) + get_token_url = OAUTH_URL + '/o/token/' + data = { + 'grant_type': 'authorization_code', + 'code': code, + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'redirect_uri': self.redirect_uri + } + result = requests.post(get_token_url, data=data) + access_token = result.json().get('access_token', None) - if access_token: - return access_token + if access_token: + return access_token - return False + return False - def get_profile(self, token): + def get_profile(self, token): - headers = { - 'Authorization': 'Bearer ' + token - } - get_profile_url = OAUTH_URL + '/api/profile/' + headers = { + 'Authorization': 'Bearer ' + token + } + get_profile_url = OAUTH_URL + '/api/profile/' - data = requests.get(get_profile_url, headers=headers).json() + data = requests.get(get_profile_url, headers=headers).json() - return data + return data From 267516d3d3e319353660e929967cce785999d6b4 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 21 Nov 2023 08:34:07 +0000 Subject: [PATCH 10/93] add more unit tests about dns service. --- tests/test_domain_register.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index d796014..ab6b792 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -30,17 +30,38 @@ authService = AuthService(logging, config.JWT_SECRET, users) dnsService = DNSService(logging, users, domains, records, ddns) -testdata = [("test.nycu-dev.me", 'A', "140.113.89.64", 5), - ("test.nycu-dev.me", 'A', "140.113.64.89", 5)] +testdata = [("test-reg.nycu-dev.me", 'A', "140.113.89.64", 5), + ("test-reg.nycu-dev.me", 'A', "140.113.64.89", 5)] answer = {"140.113.89.64", "140.113.64.89"} def test_domain_register(): - dnsService.register_domain("109550028", "test.nycu-dev.me") + dnsService.register_domain("109550028", "test-reg.nycu-dev.me") for testcase in testdata: dnsService.add_record(*testcase) time.sleep(10) - assert set(resolver.query("test.nycu-dev.me", 'A')) == answer - dnsService.release_domain("test.nycu-dev.me") + assert set(resolver.query("test-reg.nycu-dev.me", 'A')) == answer + dnsService.release_domain("test-reg.nycu-dev.me") dnsService.register_domain("109550028", "test") time.sleep(10) - assert set(resolver.query("test.nycu-dev.me", 'A')) == set() + assert set(resolver.query("test-reg.nycu-dev.me", 'A')) == set() + +def test_dulplicated_domain_register(): + dnsService.register_domain("109550028", "test-reg-dup.nycu-dev.me") + try: + dnsService.register_domain("109550028", "test-reg-dup.nycu-dev.me") + assert 0 + except Exception: + assert 1 + dnsService.release_domain("test-reg-dup.nycu-dev.me") + +def test_nxdomain_operation(): + try: + for testcase in testdata: + dnsService.add_record(*testcase) + assert 0 + except Exception: + assert 1 + try: + dnsService.release_domain("test-reg-nx.nycu-dev.me") + except Exception: + assert 1 From 5d99f39cbb8729c33fb4508c0499279ec28a68c5 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 21 Nov 2023 14:51:10 +0000 Subject: [PATCH 11/93] update config.py.sample to fit config.py --- config.py.sample | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/config.py.sample b/config.py.sample index afe634f..d8b64e5 100644 --- a/config.py.sample +++ b/config.py.sample @@ -1,19 +1,18 @@ # Oauth Parameters -NYCU_Oauth_ID = r"" -NYCU_Oauth_key = r"" -NYCU_Oauth_rURL = r"https://nycu-dev.me/oauth" +NYCU_OAUTH_ID = "" +NYCU_OAUTH_KEY = "" +NYCU_OAUTH_RURL = "" #MySQL Parameters -MySQL_Host = r"172.21.21.2" -MySQL_User = r"nycu_me" -MySQL_Pswd = r"abc123" -MySQL_DB = r"nycu_me_dns" +MYSQL_HOST = r"172.21.21.2" +MYSQL_USER = r"nycu_me" +MYSQL_PSWD = r"abc123" +MYSQL_DB = r"nycu_me_dns" #JWT Secret Key -JWT_secretKey = r"abc123" +JWT_SECRET = r"abc123" #DDNS -DDNS_KeyFile = r"/etc/ddnskey.conf" -DDNS_Server = r"172.21.21.3" -DDNS_Zone = r"nycu-dev.me" - +DDNS_KEY = r"/etc/ddnskey.conf" +DDNS_SERVER = r"172.21.21.3" +DDNS_ZONE = r"nycu-dev.me" From 1a91745e3d25718c0ed76b5e064136544bce0b7a Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 21 Nov 2023 15:34:56 +0000 Subject: [PATCH 12/93] debug: misuse of formatted string --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index b1a602c..2df9a06 100644 --- a/main.py +++ b/main.py @@ -11,7 +11,7 @@ flask_cors.CORS(app) sql_engine = create_engine( - f'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( + 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( user=config.MYSQL_USER, pswd=config.MYSQL_PSWD, host=config.MYSQL_HOST, From ef6091f3555520b344cf37c69885dcf47d4940de Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Thu, 23 Nov 2023 17:52:31 +0000 Subject: [PATCH 13/93] rename authService and dnsService to snake_case naming style, and add domain name checker. --- config.py.sample | 3 ++ services/__init__.py | 4 +-- services/{authService.py => auth_service.py} | 20 +++++++++++++ services/{dnsService.py => dns_service.py} | 31 +++++++++++++++++++- tests/test_domain_register.py | 11 +++++-- tests/test_issue_token.py | 2 +- 6 files changed, 65 insertions(+), 6 deletions(-) rename services/{authService.py => auth_service.py} (80%) rename services/{dnsService.py => dns_service.py} (74%) diff --git a/config.py.sample b/config.py.sample index d8b64e5..bf77d0c 100644 --- a/config.py.sample +++ b/config.py.sample @@ -16,3 +16,6 @@ JWT_SECRET = r"abc123" DDNS_KEY = r"/etc/ddnskey.conf" DDNS_SERVER = r"172.21.21.3" DDNS_ZONE = r"nycu-dev.me" + +#General +HOST_DOMAINS = [r"*.nycu-dev.me"] diff --git a/services/__init__.py b/services/__init__.py index 3c08951..c474b5d 100644 --- a/services/__init__.py +++ b/services/__init__.py @@ -1,3 +1,3 @@ -from .authService import * -from .dnsService import * +from .auth_service import * +from .dns_service import * from .nctu_oauth import * diff --git a/services/authService.py b/services/auth_service.py similarity index 80% rename from services/authService.py rename to services/auth_service.py index d2ef518..167b9d9 100644 --- a/services/authService.py +++ b/services/auth_service.py @@ -1,7 +1,27 @@ from sqlalchemy.orm import sessionmaker from datetime import timezone, datetime +from enum import Enum import jwt + +class UnauthorizedError(Exception): + + def __init__(self, msg): + self.msg = str(msg) + + def __str__(self): + return self.msg + + def __repr__(self): + return "Unauthorized: " + self.msg + +class OperationErrors(Enum): + NotAllowedDomain = "NotAllowedDomain" + NumberLimitExceed = "NumberLimitExceed" + AssignedDomainName = "AssignedDomainName" + PermissionDenied = "PermissionDenied" + ReservedDomain = "ReservedDomain" + class UnauthorizedError(Exception): def __init__(self, msg): self.msg = str(msg) diff --git a/services/dnsService.py b/services/dns_service.py similarity index 74% rename from services/dnsService.py rename to services/dns_service.py index a0cd971..456086e 100644 --- a/services/dnsService.py +++ b/services/dns_service.py @@ -1,7 +1,10 @@ import time +import re from enum import Enum +DOMAIN_REGEX = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(? len(struct): + return False + + for i in range(len(rule)): + if rule[i] == '*': + return 1 + elif rule[i] != struct[i]: + return False + + for domain in self.host_domains: + if is_match(domain, domain_struct): + return True def get_domain(self, domain_name): domain = self.domains.get_domain(domain_name) @@ -46,6 +73,8 @@ def get_domain(self, domain_name): return domain_info def register_domain(self, uid, domain_name): + if not self.check_domain(domain_name): + raise DNSError(DNSErrors.UNALLOWED, "This domain is not hosted by us.") domain = self.domains.get_domain(domain_name) if domain: raise DNSError(DNSErrors.UNALLOWED, "This domain have been registered.") diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index ab6b792..06b4ef2 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -28,7 +28,7 @@ records = Records(sql_engine) authService = AuthService(logging, config.JWT_SECRET, users) -dnsService = DNSService(logging, users, domains, records, ddns) +dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) testdata = [("test-reg.nycu-dev.me", 'A', "140.113.89.64", 5), ("test-reg.nycu-dev.me", 'A', "140.113.64.89", 5)] @@ -41,7 +41,7 @@ def test_domain_register(): time.sleep(10) assert set(resolver.query("test-reg.nycu-dev.me", 'A')) == answer dnsService.release_domain("test-reg.nycu-dev.me") - dnsService.register_domain("109550028", "test") + dnsService.register_domain("109550028", "test-reg.nycu-dev.me") time.sleep(10) assert set(resolver.query("test-reg.nycu-dev.me", 'A')) == set() @@ -65,3 +65,10 @@ def test_nxdomain_operation(): dnsService.release_domain("test-reg-nx.nycu-dev.me") except Exception: assert 1 + +def test_unhost_operation(): + try: + dnsService.register_domain("109550028", "www.google.com") + assert 0 + except Exception: + assert 1 diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py index 7e76fe5..c4dfcdd 100644 --- a/tests/test_issue_token.py +++ b/tests/test_issue_token.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import sessionmaker import logging -from services.authService import AuthService +from services.auth_service import AuthService from models import users, db import config From b626aa023b9dd864fa04b183010c99ca93e8a210 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Thu, 23 Nov 2023 19:23:17 +0000 Subject: [PATCH 14/93] Add domain registration validation logic in service layer. --- controllers/domains.py | 4 ++-- main.py | 9 +++++++-- services/auth_service.py | 28 ++++++++++++++++++++++++++-- services/dns_service.py | 14 +++++++------- tests/test_domain_register.py | 6 +++--- tests/test_issue_token.py | 7 ++++--- 6 files changed, 49 insertions(+), 19 deletions(-) diff --git a/controllers/domains.py b/controllers/domains.py index 7184ea6..afcbf39 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -5,8 +5,8 @@ @app.route("/domains/", methods=['POST']) def applyDomain(domain): - if not g.user: - return {"message": "Unauth."}, 401 + #if not g.user: + # return {"message": "Unauth."}, 401 user = users.getUser(g.user['uid']) domainStruct = domain.lower().strip('/').split('/') diff --git a/main.py b/main.py index 2df9a06..71498bb 100644 --- a/main.py +++ b/main.py @@ -4,7 +4,7 @@ from sqlalchemy import create_engine import config -from models import Users +from models import Users, Domains, Records, DDNS from services import AuthService, DNSService, Oauth app = Flask(__name__) @@ -19,11 +19,16 @@ ) ) +ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) + users = Users(sql_engine) +domains = Domains(sql_engine) +records = Records(sql_engine) nycu_oauth = Oauth(redirect_uri = config.NYCU_OAUTH_RURL, client_id = config.NYCU_OAUTH_ID, client_secret = config.NYCU_OAUTH_KEY) -authService = AuthService(logging, config.JWT_SECRET, users) +authService = AuthService(logging, config.JWT_SECRET, users, domains) +dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) from controllers import auth diff --git a/services/auth_service.py b/services/auth_service.py index 167b9d9..2479d37 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -22,6 +22,12 @@ class OperationErrors(Enum): PermissionDenied = "PermissionDenied" ReservedDomain = "ReservedDomain" +class Operation(Enum): + APPLY = 1 + RELEASE = 2 + MODIFY = 3 + RENEW = 4 + class UnauthorizedError(Exception): def __init__(self, msg): self.msg = str(msg) @@ -33,10 +39,11 @@ def __repr__(self): return "Unauthorized: " + self.msg class AuthService: - def __init__(self, logger, jwt_secret, users): + def __init__(self, logger, jwt_secret, users, domains): self.logger = logger self.jwt_secret = jwt_secret self.users = users + self.domains = domains def issue_token(self, profile): now = int(datetime.now(tz=timezone.utc).timestamp()) @@ -81,4 +88,21 @@ def authenticate_token(self, payload): except UnauthorizedError as e: self.logger.debug(e.__str__()) return None - + + def authorize_action(self, uid, action, domain_name): + if action == Operation.APPLY: + domains = self.domains.list_by_user(uid) + if len(domains) >= self.users.query(uid).limit: + raise OperationError(OperationErrors.NumberLimitExceed, "You cannot apply for more domains.") + if action == Operation.RELEASE: + domain = self.domains.get_domain(domain_name) + if domain.userId != uid: + raise OperationError(OperationErrors.PermissionDenied, "You cannot modify domain %s which you don't have." % (domainName, )) + if action == Operation.MODIFY: + domain = self.domains.get_domain(domain_name) + if domain.userId != uid: + raise OperationError(OperationErrors.PermissionDenied, "You cannot modify domain %s which you don't have." % (domainName, )) + if action == Operation.RENEW: + domain = self.domains.get_domain(domain_name) + if domain.userId != uid: + raise OperationError(OperationErrors.PermissionDenied, "You cannot modify domain %s which you don't have." % (domainName, )) diff --git a/services/dns_service.py b/services/dns_service.py index 456086e..0cfc025 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -35,23 +35,23 @@ def check_domain(self, domain_name): for p in domain_struct: if not DOMAIN_REGEX.fullmatch(p): - return False - + return 0 + def is_match(rule, struct): # Check if the domain is matching to a specific rule rule = list(reversed(rule.split('.'))) if len(rule) > len(struct): - return False + return 0 for i in range(len(rule)): if rule[i] == '*': - return 1 + return i + 1 elif rule[i] != struct[i]: - return False + return 0 for domain in self.host_domains: - if is_match(domain, domain_struct): - return True + if (x:=is_match(domain, domain_struct)): + return x def get_domain(self, domain_name): domain = self.domains.get_domain(domain_name) diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index 06b4ef2..fb08371 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -5,7 +5,7 @@ import time from models import Domains, Records, Users, db, DDNS -from services import AuthService, DNSService +from services import DNSService import config @@ -27,7 +27,6 @@ domains = Domains(sql_engine) records = Records(sql_engine) -authService = AuthService(logging, config.JWT_SECRET, users) dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) testdata = [("test-reg.nycu-dev.me", 'A', "140.113.89.64", 5), @@ -66,9 +65,10 @@ def test_nxdomain_operation(): except Exception: assert 1 -def test_unhost_operation(): +def test_unhost_register(): try: dnsService.register_domain("109550028", "www.google.com") assert 0 except Exception: assert 1 + diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py index c4dfcdd..f45fe3c 100644 --- a/tests/test_issue_token.py +++ b/tests/test_issue_token.py @@ -3,7 +3,7 @@ import logging from services.auth_service import AuthService -from models import users, db +from models import Users, Domains, db import config sql_engine = create_engine('sqlite:///:memory:') @@ -11,8 +11,9 @@ Session = sessionmaker(bind=sql_engine) session = Session() -users = users.Users(sql_engine) -authService = AuthService(logging, config.JWT_SECRET, users) +users = Users(sql_engine) +domains = Domains(sql_engine) +authService = AuthService(logging, config.JWT_SECRET, users, domains) testdata = [{'email':"lin.cs09@nycu.edu.tw",'username':"109550028"}, {'email':"lin.cs09@nctu.edu.tw",'username':"109550028"}] From 761ef839bd3d113003dd5573ace0b37db36f8fa3 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Fri, 24 Nov 2023 13:25:33 +0000 Subject: [PATCH 15/93] add controller domains: add function register domain --- config.py.sample | 3 +++ controllers/auth.py | 19 +++++++++++++++---- controllers/domains.py | 31 +++++++++++++++++-------------- main.py | 31 ++++++++++++++++++++++--------- models/domains.py | 3 +++ models/records.py | 2 ++ models/users.py | 3 +++ services/dns_service.py | 11 +++++++++++ tests/test_controllers.py | 18 ++++++++++++++++++ 9 files changed, 94 insertions(+), 27 deletions(-) create mode 100644 tests/test_controllers.py diff --git a/config.py.sample b/config.py.sample index bf77d0c..2ab96a7 100644 --- a/config.py.sample +++ b/config.py.sample @@ -17,5 +17,8 @@ DDNS_KEY = r"/etc/ddnskey.conf" DDNS_SERVER = r"172.21.21.3" DDNS_ZONE = r"nycu-dev.me" +#Test +TEST_PROFILE = {'email': "lin.cs09@nycu.edu.tw", + 'username': "109550028"} #General HOST_DOMAINS = [r"*.nycu-dev.me"] diff --git a/controllers/auth.py b/controllers/auth.py index ac84b10..8a317ee 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -1,18 +1,28 @@ from flask import request, g -from main import app, nycu_oauth, authService - +from main import env_test, app, nycu_oauth, authService, dnsService +import config @app.before_request def before_request(): + g.user = authService.authenticate_token(request.headers.get('Authorization')) @app.route("/oauth/", methods = ['GET']) def get_token(code): + token = nycu_oauth.get_token(code) if token: - return {"token": authService.issue_token(nycu_oauth.get_profile(token))} + return {'token': authService.issue_token(nycu_oauth.get_profile(token))} + else: + return {'message': "Invalid code."}, 401 + +@app.route("/test_auth/", methods = ['GET']) +def get_token_for_test(): + + if env_test: + return {'token': authService.issue_token(config.TEST_PROFILE)} else: - return {"message": "Invalid code."}, 401 + return {'message': "It is not currently running on testing mode."}, 401 @app.route("/whoami/", methods = ['GET']) def whoami(): @@ -20,5 +30,6 @@ def whoami(): data = {} data['uid'] = g.user['uid'] data['email'] = g.user['email'] + data['domains'] = dnsService.list_domains_by_user(g.user['uid']) return data return {"message": "Unauth."}, 401 diff --git a/controllers/domains.py b/controllers/domains.py index afcbf39..f3fb7a3 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -1,16 +1,19 @@ -from flask import Response, request -from main import app, g, authService - +from flask import Response, request, g +from main import app, authService, dnsService +from services import Operation @app.route("/domains/", methods=['POST']) -def applyDomain(domain): - - #if not g.user: - # return {"message": "Unauth."}, 401 - - user = users.getUser(g.user['uid']) - domainStruct = domain.lower().strip('/').split('/') - domainName = '.'.join(reversed(domainStruct)) - domain = dns.getDomain(domainName) - - +def register_domain(domain): + + if not g.user: + return {"message": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + #try: + authService.authorize_action(g.user['uid'], Operation.APPLY, domain_name) + dnsService.register_domain(g.user['uid'], domain_name) + return {"msg": "ok"} + #except Exception as e: + # return {"error": e.msg}, 403 diff --git a/main.py b/main.py index 71498bb..38c9c0b 100644 --- a/main.py +++ b/main.py @@ -1,23 +1,32 @@ import logging +import os from flask import Flask import flask_cors from sqlalchemy import create_engine import config -from models import Users, Domains, Records, DDNS +from models import Users, Domains, Records, DDNS, db from services import AuthService, DNSService, Oauth + +env_test = os.getenv('TEST') + app = Flask(__name__) flask_cors.CORS(app) -sql_engine = create_engine( - 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( - user=config.MYSQL_USER, - pswd=config.MYSQL_PSWD, - host=config.MYSQL_HOST, - db=config.MYSQL_DB +sql_engine = None +if env_test is not None: + sql_engine = create_engine("sqlite:///:memory:") + db.Base.metadata.create_all(sql_engine) +else: + sql_engine = create_engine( + 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( + user=config.MYSQL_USER, + pswd=config.MYSQL_PSWD, + host=config.MYSQL_HOST, + db=config.MYSQL_DB + ) ) -) ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) @@ -31,4 +40,8 @@ authService = AuthService(logging, config.JWT_SECRET, users, domains) dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) -from controllers import auth +@app.route("/") +def index(): + return "Hello World!" + +from controllers import auth, domains diff --git a/models/domains.py b/models/domains.py index f2ad882..320a667 100644 --- a/models/domains.py +++ b/models/domains.py @@ -10,18 +10,21 @@ def __init__(self, sql_engine): self.session = Session() def get_domain(self, domain): + self.session.expire_all() domain = self.session.query(db.Domain).filter_by(domain=domain, status=1).all() if domain: return domain[0] return None def get_domain_by_id(self, domain_id): + self.session.expire_all() domain = self.session.query(db.Domain).filter_by(id=domain_id, status=1).all() if domain: return domain[0] return None def list_by_user(self, user): + self.session.expire_all() domains = self.session.query(db.Domain).filter_by(userId=user, status=1).all() return domains diff --git a/models/records.py b/models/records.py index cfce1c1..12f4aa5 100644 --- a/models/records.py +++ b/models/records.py @@ -10,9 +10,11 @@ def __init__(self, sql_engine): self.session = Session() def get_record(self, record_id): + self.session.expire_all() return self.session.query(db.Record).filter_by(id=record_id, status=1).first() def get_records(self, domain_id): + self.session.expire_all() return self.session.query(db.Record).filter_by(domain_id=domain_id, status=1).all() def add_record(self, domain_id, record_type, value, ttl): diff --git a/models/users.py b/models/users.py index 092a3c7..db4cfe8 100644 --- a/models/users.py +++ b/models/users.py @@ -1,12 +1,15 @@ from sqlalchemy.orm import sessionmaker from . import db import logging + + class Users: def __init__(self, sql_engine): Session = sessionmaker(bind=sql_engine) self.session = Session() def query(self, uid): + self.session.expire_all() user = self.session.query(db.User).filter_by(id=uid).all() if len(user): return user[0] diff --git a/services/dns_service.py b/services/dns_service.py index 0cfc025..b477f5d 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -72,6 +72,17 @@ def get_domain(self, domain_name): record.ttl)) return domain_info + def list_domains_by_user(self, uid): + domains = [] + for domain in self.domains.list_by_user(uid): + domain_info = {} + domain_info['id'] = domain.id + domain_info['regDate'] = domain.regDate + domain_info['expDate'] = domain.expDate + domain_info['domain'] = domain.domain + domains.append(domain_info) + return domains + def register_domain(self, uid, domain_name): if not self.check_domain(domain_name): raise DNSError(DNSErrors.UNALLOWED, "This domain is not hosted by us.") diff --git a/tests/test_controllers.py b/tests/test_controllers.py new file mode 100644 index 0000000..b1c300c --- /dev/null +++ b/tests/test_controllers.py @@ -0,0 +1,18 @@ +import requests +import json +import config +import time + + +URL_BASE = "http://172.21.21.4:8000/" + +response = requests.get(URL_BASE + "test_auth/") +token = json.loads(response.text)['token'] +headers = {'Authorization': 'Bearer ' + token} + +def test_register_domain(): + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-route-reg", headers = headers) + assert response.status_code == 200 + response = requests.get(URL_BASE + "/whoami/", headers = headers) + domains = json.loads(response.text)['domains'] + assert {domain['domain'] for domain in domains} == {'test-route-reg.nycu-dev.me'} From 75ffd6a3688c81bfc7db70b9f5301bc97bfcf354 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Fri, 24 Nov 2023 17:10:03 +0000 Subject: [PATCH 16/93] add controller ddns: add function add/delete records --- config.py.sample | 3 -- controllers/__init__.py | 2 + controllers/auth.py | 2 +- controllers/ddns.py | 96 +++++++++++++++++++++++++++++++++++ controllers/domains.py | 28 +++++++--- main.py | 2 +- models/records.py | 8 ++- services/auth_service.py | 26 ++-------- services/dns_service.py | 15 ++++-- tests/test_controllers.py | 62 +++++++++++++++++++--- tests/test_domain_register.py | 11 +++- tests/test_permission.py | 42 +++++++++++++++ 12 files changed, 252 insertions(+), 45 deletions(-) create mode 100644 controllers/ddns.py create mode 100644 tests/test_permission.py diff --git a/config.py.sample b/config.py.sample index 2ab96a7..bf77d0c 100644 --- a/config.py.sample +++ b/config.py.sample @@ -17,8 +17,5 @@ DDNS_KEY = r"/etc/ddnskey.conf" DDNS_SERVER = r"172.21.21.3" DDNS_ZONE = r"nycu-dev.me" -#Test -TEST_PROFILE = {'email': "lin.cs09@nycu.edu.tw", - 'username': "109550028"} #General HOST_DOMAINS = [r"*.nycu-dev.me"] diff --git a/controllers/__init__.py b/controllers/__init__.py index 35c1920..d1ebdc1 100644 --- a/controllers/__init__.py +++ b/controllers/__init__.py @@ -1 +1,3 @@ from .auth import * +from .domains import * +from .ddns import * diff --git a/controllers/auth.py b/controllers/auth.py index 8a317ee..00ecbe1 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -20,7 +20,7 @@ def get_token(code): def get_token_for_test(): if env_test: - return {'token': authService.issue_token(config.TEST_PROFILE)} + return {'token': authService.issue_token(request.json)} else: return {'message': "It is not currently running on testing mode."}, 401 diff --git a/controllers/ddns.py b/controllers/ddns.py new file mode 100644 index 0000000..94185fa --- /dev/null +++ b/controllers/ddns.py @@ -0,0 +1,96 @@ +from flask import Response, request, g +import re, ipaddress + +from main import app, authService, dnsService +from services import Operation + + + +domainRegex = re.compile(r'^([A-Za-z0-9]\.|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]\.){1,3}[A-Za-z]{2,6}$') + +def is_ip(addr, protocol = ipaddress.IPv4Address): + try: + ip = ipaddress.ip_address(addr) + if isinstance(ip, protocol): + return str(ip) + return False + except: + return False + +def is_domain(domain): + return domainRegex.fullmatch(domain) + +def check_type(type_, value): + + if type_ == 'A': + if not is_ip(value, ipaddress.IPv4Address): + return {"errorType": "DNSError", "msg": "Type A with non-IPv4 value."}, 403 + + value = is_ip(value, ipaddress.IPv4Address) + + if type_ == 'AAAA': + if not is_ip(value, ipaddress.IPv6Address): + return {"errorType": "DNSError", "msg": "Type AAAA with non-IPv6 value."}, 403 + + value = is_ip(value, ipaddress.IPv6Address) + + if type_ == 'CNAME' and not is_domain(value): + return {"errorType": "DNSError", "msg": "Type CNAME with non-domain-name value."}, 403 + + if type_ == 'MX' and not is_domain(value): + return {"errorType": "DNSError", "msg": "Type MX with non-domain-name value."}, 403 + + if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): + return {"errorType": "DNSError", "msg": "Type TXT with value longer than 255 chars or more than 1 line."}, 403 + + return None + + +@app.route("/ddns//records//", methods=['POST']) +def addRecord(domain, type_, value): + + if not g.user: + return {"message": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + try: + req = request.json + if req and 'ttl' in req and req['ttl'].isnumeric() and 5 <= int(req['ttl']) <= 86400: + ttl = int(req['ttl']) + else: + ttl = 5 + except: + ttl = 5 + + check_result = check_type(type_, value) + if check_result: + return check_result + + try: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + dnsService.add_record(domain_name, type_, value, ttl) + return {"msg": "ok"} + except Exception as e: + return {"error": e.msg}, 403 + +@app.route("/ddns//records//", methods=['DELETE']) +def delRecord(domain, type_, value): + + if not g.user: + return {"message": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + check_result = check_type(type_, value) + if check_result: + return check_result + + try: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + dnsService.del_record(domain_name, type_, value) + return {"msg": "ok"} + except Exception as e: + return {"error": e.msg}, 403 diff --git a/controllers/domains.py b/controllers/domains.py index f3fb7a3..afadc45 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -11,9 +11,25 @@ def register_domain(domain): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) - #try: - authService.authorize_action(g.user['uid'], Operation.APPLY, domain_name) - dnsService.register_domain(g.user['uid'], domain_name) - return {"msg": "ok"} - #except Exception as e: - # return {"error": e.msg}, 403 + try: + authService.authorize_action(g.user['uid'], Operation.APPLY, domain_name) + dnsService.register_domain(g.user['uid'], domain_name) + return {"msg": "ok"} + except Exception as e: + return {"error": e.msg}, 403 + +@app.route("/domains/", methods=['DELETE']) +def release_domain(domain): + + if not g.user: + return {"message": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + try: + authService.authorize_action(g.user['uid'], Operation.RELEASE, domain_name) + dnsService.release_domain(domain_name) + return {"msg": "ok"} + except Exception as e: + return {"error": e.msg}, 403 diff --git a/main.py b/main.py index 38c9c0b..488999f 100644 --- a/main.py +++ b/main.py @@ -44,4 +44,4 @@ def index(): return "Hello World!" -from controllers import auth, domains +from controllers import auth, domains, ddns diff --git a/models/records.py b/models/records.py index 12f4aa5..6b7ed8b 100644 --- a/models/records.py +++ b/models/records.py @@ -17,6 +17,12 @@ def get_records(self, domain_id): self.session.expire_all() return self.session.query(db.Record).filter_by(domain_id=domain_id, status=1).all() + def get_record_by_type_value(self, domain_id, type_, value): + return self.session.query(db.Record).filter_by(domain_id=domain_id, + type=type_, + value=value, + status=1).first() + def add_record(self, domain_id, record_type, value, ttl): now = datetime.now() regDate = now @@ -29,7 +35,7 @@ def add_record(self, domain_id, record_type, value, ttl): self.session.add(record) self.session.commit() - def del_record(self, record_id): + def del_record_by_id(self, record_id): records = self.session.query(db.Record).filter_by(id=record_id).all() now = datetime.now() for record in records: diff --git a/services/auth_service.py b/services/auth_service.py index 2479d37..259b18d 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -4,24 +4,6 @@ import jwt -class UnauthorizedError(Exception): - - def __init__(self, msg): - self.msg = str(msg) - - def __str__(self): - return self.msg - - def __repr__(self): - return "Unauthorized: " + self.msg - -class OperationErrors(Enum): - NotAllowedDomain = "NotAllowedDomain" - NumberLimitExceed = "NumberLimitExceed" - AssignedDomainName = "AssignedDomainName" - PermissionDenied = "PermissionDenied" - ReservedDomain = "ReservedDomain" - class Operation(Enum): APPLY = 1 RELEASE = 2 @@ -93,16 +75,16 @@ def authorize_action(self, uid, action, domain_name): if action == Operation.APPLY: domains = self.domains.list_by_user(uid) if len(domains) >= self.users.query(uid).limit: - raise OperationError(OperationErrors.NumberLimitExceed, "You cannot apply for more domains.") + raise UnauthorizedError("You cannot apply for more domains.") if action == Operation.RELEASE: domain = self.domains.get_domain(domain_name) if domain.userId != uid: - raise OperationError(OperationErrors.PermissionDenied, "You cannot modify domain %s which you don't have." % (domainName, )) + raise UnauthorizedError("You cannot modify domain %s which you don't have." % (domain_name, )) if action == Operation.MODIFY: domain = self.domains.get_domain(domain_name) if domain.userId != uid: - raise OperationError(OperationErrors.PermissionDenied, "You cannot modify domain %s which you don't have." % (domainName, )) + raise UnauthorizedError("You cannot modify domain %s which you don't have." % (domain_name, )) if action == Operation.RENEW: domain = self.domains.get_domain(domain_name) if domain.userId != uid: - raise OperationError(OperationErrors.PermissionDenied, "You cannot modify domain %s which you don't have." % (domainName, )) + raise UnauthorizedError("You cannot modify domain %s which you don't have." % (domain_name, )) diff --git a/services/dns_service.py b/services/dns_service.py index b477f5d..83e5d8a 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -103,7 +103,7 @@ def release_domain(self, domain_name): raise DNSError(DNSErrors.NXDOMAIN, "This domain is not registered.") records = self.records.get_records(domain.id) for record in records: - self.del_record(record.id) + self.del_record_by_id(record.id) self.domains.release(domain_name) def add_record(self, domain_name, type_, value, ttl): @@ -118,12 +118,21 @@ def add_record(self, domain_name, type_, value, ttl): self.records.add_record(domain.id, type_, value, ttl) self.ddns.add_record(domain_name, type_, value, ttl) + + def del_record(self, domain_name, type_, value): + domain_id = self.domains.get_domain(domain_name).id + record = self.records.get_record_by_type_value(domain_id, + type_, + value) + if not record: + raise DNSError(DNSErrors.UNALLOWED, "This record does not exist.") + self.del_record_by_id(record.id) - def del_record(self, record_id): + def del_record_by_id(self, record_id): record = self.records.get_record(record_id) if not record: raise DNSError(DNSErrors.UNALLOWED, "This record does not exist.") domain = self.domains.get_domain_by_id(record.domain_id) - self.records.del_record(record_id) + self.records.del_record_by_id(record_id) self.ddns.del_record(domain.domain, record.type, record.value) diff --git a/tests/test_controllers.py b/tests/test_controllers.py index b1c300c..d49ba3e 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -2,17 +2,65 @@ import json import config import time - +import pydig URL_BASE = "http://172.21.21.4:8000/" -response = requests.get(URL_BASE + "test_auth/") -token = json.loads(response.text)['token'] -headers = {'Authorization': 'Bearer ' + token} +def get_headers(uid): + data = { + "email": "lin.cs09@nycu.edu.tw", + "username": uid + } + response = requests.get(URL_BASE + "test_auth/", json = data) + token = json.loads(response.text)['token'] + return {'Authorization': 'Bearer ' + token} + +resolver = pydig.Resolver( + executable='/usr/bin/dig', + nameservers=[ + '172.21.21.3' + ], +) + +def test_register_and_release_domain(): + headers = get_headers("109550004") + # Register domains + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-route-reg1", headers = headers) + assert response.status_code == 200 + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-route-reg2", headers = headers) + assert response.status_code == 200 + + # check if the entries exist + response = requests.get(URL_BASE + "/whoami/", headers = headers) + domains = json.loads(response.text)['domains'] + assert {domain['domain'] for domain in domains} == {'test-route-reg1.nycu-dev.me', 'test-route-reg2.nycu-dev.me'} -def test_register_domain(): - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-route-reg", headers = headers) + # release the domains + response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-route-reg1", headers = headers) assert response.status_code == 200 + response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-route-reg2", headers = headers) + assert response.status_code == 200 + + # check if the domain were released response = requests.get(URL_BASE + "/whoami/", headers = headers) domains = json.loads(response.text)['domains'] - assert {domain['domain'] for domain in domains} == {'test-route-reg.nycu-dev.me'} + assert {domain['domain'] for domain in domains} == set() + +def test_add_and_delete_records(): + headers = get_headers("109550028") + # Register domains + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-route-rec", headers = headers) + assert response.status_code == 200 + # Add records + response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.89.64", headers = headers) + assert response.status_code == 200 + response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.64.89", headers = headers) + assert response.status_code == 200 + # Check the result + time.sleep(10) + assert set(resolver.query("test-route-rec.nycu-dev.me", 'A')) == {"140.113.89.64", "140.113.64.89"} + # Remove the records + response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.89.64", headers = headers) + assert response.status_code == 200 + response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.64.89", headers = headers) + assert response.status_code == 200 diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index fb08371..c59afe9 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -44,7 +44,7 @@ def test_domain_register(): time.sleep(10) assert set(resolver.query("test-reg.nycu-dev.me", 'A')) == set() -def test_dulplicated_domain_register(): +def test_duplicated_domain_register(): dnsService.register_domain("109550028", "test-reg-dup.nycu-dev.me") try: dnsService.register_domain("109550028", "test-reg-dup.nycu-dev.me") @@ -72,3 +72,12 @@ def test_unhost_register(): except Exception: assert 1 +def test_duplicated_record(): + dnsService.register_domain("109550028", "test-add-dup-rec.nycu-dev.me") + dnsService.add_record("test-add-dup-rec.nycu-dev.me", 'A', "140.113.64.89", 5) + try: + dnsService.add_record("test-add-dup-rec.nycu-dev.me", 'A', "140.113.64.89", 5) + assert 0 + except Exception: + assert 1 + dnsService.release_domain("test-add-dup-rec.nycu-dev.me") diff --git a/tests/test_permission.py b/tests/test_permission.py new file mode 100644 index 0000000..a3e0588 --- /dev/null +++ b/tests/test_permission.py @@ -0,0 +1,42 @@ +import requests +import json + +URL_BASE = "http://172.21.21.4:8000/" + +def get_headers(uid): + data = { + "email": "lin.cs09@nycu.edu.tw", + "username": uid + } + response = requests.get(URL_BASE + "test_auth/", json = data) + token = json.loads(response.text)['token'] + return {'Authorization': 'Bearer ' + token} + +def test_permission(): + h04 = get_headers("109550004") + h28 = get_headers("109550028") + + # Register domain + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h28) + assert response.status_code == 200 + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h04) + assert response.status_code == 403 + + # Add record + response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h04) + assert response.status_code == 403 + response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h28) + assert response.status_code == 200 + + # Delete record + response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h04) + assert response.status_code == 403 + response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h28) + assert response.status_code == 200 + + # Release domain + response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h04) + assert response.status_code == 403 + response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h28) + assert response.status_code == 200 + From d84671eb79c443a89c5ea4698cb1bcf68d7a32f5 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 27 Nov 2023 11:07:09 +0000 Subject: [PATCH 17/93] Not using one session. Generate a new session each time instead. --- models/domains.py | 117 +++++++++++++++++++++++++++------------------- models/records.py | 78 ++++++++++++++++++++----------- models/users.py | 57 +++++++++++++--------- 3 files changed, 153 insertions(+), 99 deletions(-) diff --git a/models/domains.py b/models/domains.py index 320a667..5994d29 100644 --- a/models/domains.py +++ b/models/domains.py @@ -1,57 +1,78 @@ -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import sessionmaker, scoped_session from datetime import datetime, timedelta - from . import db - +import logging class Domains: def __init__(self, sql_engine): - Session = sessionmaker(bind=sql_engine) - self.session = Session() + self.sql_engine = sql_engine + self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) - def get_domain(self, domain): - self.session.expire_all() - domain = self.session.query(db.Domain).filter_by(domain=domain, status=1).all() - if domain: - return domain[0] - return None + def get_domain(self, domain_name): + session = self.Session() + try: + domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() + return domain + finally: + session.close() def get_domain_by_id(self, domain_id): - self.session.expire_all() - domain = self.session.query(db.Domain).filter_by(id=domain_id, status=1).all() - if domain: - return domain[0] - return None - - def list_by_user(self, user): - self.session.expire_all() - domains = self.session.query(db.Domain).filter_by(userId=user, status=1).all() - return domains - - def register(self, domain, user): - now = datetime.now() - regDate = now - expDate = (now + timedelta(days=30)) - domain = db.Domain(userId = user, - domain = domain, - regDate = regDate, - expDate = expDate, - status = 1) - self.session.add(domain) - self.session.commit() - - def renew(self, domain): - domain = self.get_domain(domain) - if domain != None: + session = self.Session() + try: + domain = session.query(db.Domain).filter_by(id=domain_id, status=1).first() + return domain + finally: + session.close() + + def list_by_user(self, user_id): + session = self.Session() + try: + domains = session.query(db.Domain).filter_by(userId=user_id, status=1).all() + return domains + finally: + session.close() + + def register(self, domain_name, user_id): + session = self.Session() + try: now = datetime.now() - expDate = (now + timedelta(days=30)) - domain.expDate = expDate - self.session.commit() - - def release(self, domain): - domain = self.get_domain(domain) - if domain != None: - expDate = datetime.now() - domain.expDate = expDate - domain.status = 0 - self.session.commit() + domain = db.Domain(userId=user_id, + domain=domain_name, + regDate=now, + expDate=now + timedelta(days=30), + status=1) + session.add(domain) + session.commit() + except Exception as e: + logging.error(f"Error registering domain: {e}") + session.rollback() + finally: + session.close() + + def renew(self, domain_name): + session = self.Session() + try: + domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() + if domain: + domain.expDate = datetime.now() + timedelta(days=30) + session.commit() + except Exception as e: + logging.error(f"Error renewing domain: {e}") + session.rollback() + finally: + session.close() + + def release(self, domain_name): + session = self.Session() + try: + domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() + if domain: + domain.expDate = datetime.now() + domain.status = 0 + session.commit() + except Exception as e: + logging.error(f"Error releasing domain: {e}") + session.rollback() + finally: + session.close() + diff --git a/models/records.py b/models/records.py index 6b7ed8b..12e3673 100644 --- a/models/records.py +++ b/models/records.py @@ -1,4 +1,4 @@ -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import sessionmaker, scoped_session from datetime import datetime from . import db @@ -6,39 +6,61 @@ class Records: def __init__(self, sql_engine): - Session = sessionmaker(bind=sql_engine) - self.session = Session() + self.sql_engine = sql_engine + self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) def get_record(self, record_id): - self.session.expire_all() - return self.session.query(db.Record).filter_by(id=record_id, status=1).first() - + session = self.Session() + try: + return session.query(db.Record).filter_by(id=record_id, status=1).first() + finally: + session.close() + def get_records(self, domain_id): - self.session.expire_all() - return self.session.query(db.Record).filter_by(domain_id=domain_id, status=1).all() + session = self.Session() + try: + return session.query(db.Record).filter_by(domain_id=domain_id, status=1).all() + finally: + session.close() def get_record_by_type_value(self, domain_id, type_, value): - return self.session.query(db.Record).filter_by(domain_id=domain_id, - type=type_, - value=value, - status=1).first() + session = self.Session() + try: + return session.query(db.Record).filter_by(domain_id=domain_id, + type=type_, + value=value, + status=1).first() + finally: + session.close() def add_record(self, domain_id, record_type, value, ttl): - now = datetime.now() - regDate = now - record = db.Record(domain_id=domain_id, - type=record_type, - value=value, - ttl=ttl, - regDate=regDate, - status=1) - self.session.add(record) - self.session.commit() + session = self.Session() + try: + record = db.Record(domain_id=domain_id, + type=record_type, + value=value, + ttl=ttl, + regDate=datetime.now(), + status=1) + session.add(record) + session.commit() + except Exception as e: + session.rollback() + raise e + finally: + session.close() def del_record_by_id(self, record_id): - records = self.session.query(db.Record).filter_by(id=record_id).all() - now = datetime.now() - for record in records: - record.status = 0 - record.expDate = now - self.session.commit() + session = self.Session() + try: + record = session.query(db.Record).filter_by(id=record_id).first() + if record: + record.status = 0 + record.expDate = datetime.now() + session.commit() + except Exception as e: + session.rollback() + raise e + finally: + session.close() + diff --git a/models/users.py b/models/users.py index db4cfe8..4011323 100644 --- a/models/users.py +++ b/models/users.py @@ -1,34 +1,45 @@ -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import sessionmaker, scoped_session from . import db import logging - class Users: def __init__(self, sql_engine): - Session = sessionmaker(bind=sql_engine) - self.session = Session() - + self.sql_engine = sql_engine + self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) + def query(self, uid): - self.session.expire_all() - user = self.session.query(db.User).filter_by(id=uid).all() - if len(user): - return user[0] - return None + session = self.Session() + try: + user = session.query(db.User).filter_by(id=uid).first() + return user + except Exception as e: + logging.error(f"Error querying user: {e}") + return None + finally: + session.close() def add(self, uid, name, username, password, status, email): - user = db.User(id=uid, - name=name, - username=username, - password=password, - status=status, - email=email) - self.session.add(user) - self.session.commit() + session = self.Session() + try: + user = db.User(id=uid, name=name, username=username, password=password, status=status, email=email) + session.add(user) + session.commit() + except Exception as e: + logging.error(f"Error adding user: {e}") + session.rollback() + finally: + session.close() def update_email(self, uid, email): - user = self.query(uid) - user.email = email - self.session.commit() + session = self.Session() + try: + user = session.query(db.User).filter_by(id=uid).first() + if user: + user.email = email + session.commit() + except Exception as e: + logging.error(f"Error updating user email: {e}") + session.rollback() + finally: + session.close() - def __del__(self): - self.session.close() From 52955b7cc068a378565a84ff2fa9f785bc6136ff Mon Sep 17 00:00:00 2001 From: roger Date: Mon, 27 Nov 2023 21:27:42 +0800 Subject: [PATCH 18/93] improve pylint score --- .pylintrc | 595 ++++++++++++++++++++++++++++++++++++++ models/db.py | 6 +- models/ddns.py | 29 +- models/domains.py | 14 +- models/records.py | 10 +- tests/test_common.py | 13 + tests/test_controllers.py | 14 +- tests/test_issue_token.py | 8 +- tests/test_permission.py | 13 +- 9 files changed, 643 insertions(+), 59 deletions(-) create mode 100644 .pylintrc create mode 100644 tests/test_common.py diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..c7af0b8 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,595 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape, + duplicate-code, + missing-function-docstring, + missing-module-docstring, + missing-class-docstring + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/models/db.py b/models/db.py index f6b5f85..f167609 100644 --- a/models/db.py +++ b/models/db.py @@ -1,6 +1,6 @@ -from sqlalchemy import create_engine, Column, String, Integer, DateTime, ForeignKey, Text, CHAR, BOOLEAN, UniqueConstraint -from sqlalchemy.orm import declarative_base -from sqlalchemy.orm import relationship +from sqlalchemy import Column, String,\ + Integer, DateTime, ForeignKey, Text, CHAR, BOOLEAN +from sqlalchemy.orm import declarative_base, relationship Base = declarative_base() diff --git a/models/ddns.py b/models/ddns.py index 491b5e8..533a762 100644 --- a/models/ddns.py +++ b/models/ddns.py @@ -4,25 +4,23 @@ import time import logging -verbose = 1 - class DDNS: def __launch(self): pr = subprocess.Popen( - ['nsupdate', '-k', self.keyFile], + ['nsupdate', '-k', self.key_file], bufsize = 0, stdin = subprocess.PIPE, stdout = subprocess.PIPE) - - if self.nServer: - pr.stdin.write(f"server {self.nServer}\n".encode()) - if self.zone: + if self.name_server: + pr.stdin.write(f"server {self.name_server}\n".encode()) + + if self.zone: pr.stdin.write(f"zone {self.zone}\n".encode()) - + return pr - + def __write(self): diff = 0 while True: @@ -31,27 +29,27 @@ def __write(self): cmd = self.queue.get() self.nsupdate.stdin.write((cmd + "\n").encode()) diff = 1 - logging.debug("executing command: {cmd}".format(cmd=cmd)); + logging.debug("executing command: {cmd}".format(cmd=cmd)) if self.nsupdate.poll(): self.queue.put(cmd) self.nsupdate = self.__launch() logging.warning("Subprocess nsupdate is dead.") - + if diff and self.nsupdate.poll() == None: diff = 0 self.nsupdate.stdin.write(b"send\n") - + except Exception as e: logging.warning(e) raise Exception(e) time.sleep(5) - def __init__(self, logger, keyFile, nServer, zone): + def __init__(self, logger, key_file, name_server, zone): self.logger = logger - self.keyFile = keyFile - self.nServer = nServer + self.key_file = key_file + self.name_server = name_server self.zone = zone self.nsupdate = self.__launch() @@ -70,4 +68,3 @@ def del_record(self, domain, rectype, value): if rectype == "TXT": value = '"%s"' % value.replace('"', '\"') self.queue.put("update delete %s %s %s" % (domain, rectype, value)) - diff --git a/models/domains.py b/models/domains.py index 5994d29..4c111df 100644 --- a/models/domains.py +++ b/models/domains.py @@ -6,10 +6,10 @@ class Domains: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) + self.make_session = scoped_session(sessionmaker(bind=self.sql_engine)) def get_domain(self, domain_name): - session = self.Session() + session = self.make_session() try: domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() return domain @@ -17,7 +17,7 @@ def get_domain(self, domain_name): session.close() def get_domain_by_id(self, domain_id): - session = self.Session() + session = self.make_session() try: domain = session.query(db.Domain).filter_by(id=domain_id, status=1).first() return domain @@ -25,7 +25,7 @@ def get_domain_by_id(self, domain_id): session.close() def list_by_user(self, user_id): - session = self.Session() + session = self.make_session() try: domains = session.query(db.Domain).filter_by(userId=user_id, status=1).all() return domains @@ -33,7 +33,7 @@ def list_by_user(self, user_id): session.close() def register(self, domain_name, user_id): - session = self.Session() + session = self.make_session() try: now = datetime.now() domain = db.Domain(userId=user_id, @@ -50,7 +50,7 @@ def register(self, domain_name, user_id): session.close() def renew(self, domain_name): - session = self.Session() + session = self.make_session() try: domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() if domain: @@ -63,7 +63,7 @@ def renew(self, domain_name): session.close() def release(self, domain_name): - session = self.Session() + session = self.make_session() try: domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() if domain: diff --git a/models/records.py b/models/records.py index 12e3673..37df686 100644 --- a/models/records.py +++ b/models/records.py @@ -1,9 +1,8 @@ -from sqlalchemy.orm import sessionmaker, scoped_session from datetime import datetime +from sqlalchemy.orm import sessionmaker, scoped_session from . import db - class Records: def __init__(self, sql_engine): self.sql_engine = sql_engine @@ -26,9 +25,9 @@ def get_records(self, domain_id): def get_record_by_type_value(self, domain_id, type_, value): session = self.Session() try: - return session.query(db.Record).filter_by(domain_id=domain_id, - type=type_, - value=value, + return session.query(db.Record).filter_by(domain_id=domain_id, + type=type_, + value=value, status=1).first() finally: session.close() @@ -63,4 +62,3 @@ def del_record_by_id(self, record_id): raise e finally: session.close() - diff --git a/tests/test_common.py b/tests/test_common.py new file mode 100644 index 0000000..4def5f7 --- /dev/null +++ b/tests/test_common.py @@ -0,0 +1,13 @@ +import requests +import json + +URL_BASE = "http://172.21.21.4:8000/" + +def get_headers(uid): + data = { + "email": "lin.cs09@nycu.edu.tw", + "username": uid + } + response = requests.get(URL_BASE + "test_auth/", json = data) + token = json.loads(response.text)['token'] + return {'Authorization': 'Bearer ' + token} \ No newline at end of file diff --git a/tests/test_controllers.py b/tests/test_controllers.py index d49ba3e..ffe0b56 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -1,19 +1,11 @@ -import requests import json -import config import time +import requests import pydig +from .test_common import get_headers, URL_BASE + -URL_BASE = "http://172.21.21.4:8000/" -def get_headers(uid): - data = { - "email": "lin.cs09@nycu.edu.tw", - "username": uid - } - response = requests.get(URL_BASE + "test_auth/", json = data) - token = json.loads(response.text)['token'] - return {'Authorization': 'Bearer ' + token} resolver = pydig.Resolver( executable='/usr/bin/dig', diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py index f45fe3c..d4bdcc1 100644 --- a/tests/test_issue_token.py +++ b/tests/test_issue_token.py @@ -6,7 +6,7 @@ from models import Users, Domains, db import config -sql_engine = create_engine('sqlite:///:memory:') +sql_engine = create_engine('sqlite:///:memory:') db.Base.metadata.create_all(sql_engine) Session = sessionmaker(bind=sql_engine) session = Session() @@ -21,10 +21,10 @@ def test_issue_token(): for testcase in testdata: token = "Bearer " + authService.issue_token(testcase) - assert authService.authenticate_token(token) != None + assert authService.authenticate_token(token) is not None # test modified token - assert authService.authenticate_token(token + 'a') == None + assert authService.authenticate_token(token + 'a') == None # test if data is written - session.expire_all() # flush orm cache + session.expire_all()# flush orm cache data = session.query(db.User).filter_by(id=testcase['username']).all() assert len(data) and data[0].email == testcase['email'] diff --git a/tests/test_permission.py b/tests/test_permission.py index a3e0588..488827b 100644 --- a/tests/test_permission.py +++ b/tests/test_permission.py @@ -1,16 +1,5 @@ import requests -import json - -URL_BASE = "http://172.21.21.4:8000/" - -def get_headers(uid): - data = { - "email": "lin.cs09@nycu.edu.tw", - "username": uid - } - response = requests.get(URL_BASE + "test_auth/", json = data) - token = json.loads(response.text)['token'] - return {'Authorization': 'Bearer ' + token} +from .test_common import get_headers, URL_BASE def test_permission(): h04 = get_headers("109550004") From f2a38a2d6c028fdd1c3b38779ff0c8d043377e7d Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Mon, 27 Nov 2023 08:57:39 -0500 Subject: [PATCH 19/93] fix typo: from domain_id to domain in models. --- models/db.py | 5 +---- models/records.py | 6 +++--- services/dns_service.py | 6 +----- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/models/db.py b/models/db.py index f167609..ea08367 100644 --- a/models/db.py +++ b/models/db.py @@ -29,14 +29,11 @@ class Domain(Base): expDate = Column(DateTime) status = Column(BOOLEAN, default=True) - # Relationships - records = relationship("Record", backref="domain") - class Record(Base): __tablename__ = 'records' id = Column(Integer, primary_key=True, autoincrement=True) - domain_id = Column(Integer, ForeignKey('domains.id'), nullable=False) + domain = Column(Integer, ForeignKey('domains.id'), nullable=False) type = Column(CHAR(16), nullable=False) value = Column(String(256)) ttl = Column(Integer, nullable=False) diff --git a/models/records.py b/models/records.py index 37df686..d45064e 100644 --- a/models/records.py +++ b/models/records.py @@ -18,14 +18,14 @@ def get_record(self, record_id): def get_records(self, domain_id): session = self.Session() try: - return session.query(db.Record).filter_by(domain_id=domain_id, status=1).all() + return session.query(db.Record).filter_by(domain=domain_id, status=1).all() finally: session.close() def get_record_by_type_value(self, domain_id, type_, value): session = self.Session() try: - return session.query(db.Record).filter_by(domain_id=domain_id, + return session.query(db.Record).filter_by(domain=domain_id, type=type_, value=value, status=1).first() @@ -35,7 +35,7 @@ def get_record_by_type_value(self, domain_id, type_, value): def add_record(self, domain_id, record_type, value, ttl): session = self.Session() try: - record = db.Record(domain_id=domain_id, + record = db.Record(domain=domain_id, type=record_type, value=value, ttl=ttl, diff --git a/services/dns_service.py b/services/dns_service.py index 83e5d8a..e4b6f3c 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -75,11 +75,7 @@ def get_domain(self, domain_name): def list_domains_by_user(self, uid): domains = [] for domain in self.domains.list_by_user(uid): - domain_info = {} - domain_info['id'] = domain.id - domain_info['regDate'] = domain.regDate - domain_info['expDate'] = domain.expDate - domain_info['domain'] = domain.domain + domain_info = self.get_domain(domain.domain) domains.append(domain_info) return domains From 3e9f6067bbc5813a51fc04a1c23079e85c6cb29b Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Mon, 27 Nov 2023 10:26:30 -0500 Subject: [PATCH 20/93] modify return message --- controllers/auth.py | 6 +++--- controllers/ddns.py | 4 ++-- controllers/domains.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/controllers/auth.py b/controllers/auth.py index 00ecbe1..f145541 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -14,15 +14,15 @@ def get_token(code): if token: return {'token': authService.issue_token(nycu_oauth.get_profile(token))} else: - return {'message': "Invalid code."}, 401 + return {'msg': "Invalid code."}, 401 @app.route("/test_auth/", methods = ['GET']) def get_token_for_test(): if env_test: - return {'token': authService.issue_token(request.json)} + return {'msg': 'ok', 'token': authService.issue_token(request.json)} else: - return {'message': "It is not currently running on testing mode."}, 401 + return {'msg': "It is not currently running on testing mode."}, 401 @app.route("/whoami/", methods = ['GET']) def whoami(): diff --git a/controllers/ddns.py b/controllers/ddns.py index 94185fa..a46092a 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -73,7 +73,7 @@ def addRecord(domain, type_, value): dnsService.add_record(domain_name, type_, value, ttl) return {"msg": "ok"} except Exception as e: - return {"error": e.msg}, 403 + return {"msg": e.msg}, 403 @app.route("/ddns//records//", methods=['DELETE']) def delRecord(domain, type_, value): @@ -93,4 +93,4 @@ def delRecord(domain, type_, value): dnsService.del_record(domain_name, type_, value) return {"msg": "ok"} except Exception as e: - return {"error": e.msg}, 403 + return {"msg": e.msg}, 403 diff --git a/controllers/domains.py b/controllers/domains.py index afadc45..2476bb0 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -16,7 +16,7 @@ def register_domain(domain): dnsService.register_domain(g.user['uid'], domain_name) return {"msg": "ok"} except Exception as e: - return {"error": e.msg}, 403 + return {"msg": e.msg}, 403 @app.route("/domains/", methods=['DELETE']) def release_domain(domain): @@ -32,4 +32,4 @@ def release_domain(domain): dnsService.release_domain(domain_name) return {"msg": "ok"} except Exception as e: - return {"error": e.msg}, 403 + return {"msg": e.msg}, 403 From 2e389e05f620b1cc9c8d9a9ba539c573e1ff91a0 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Mon, 27 Nov 2023 10:37:27 -0500 Subject: [PATCH 21/93] debug: fix typo --- services/dns_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dns_service.py b/services/dns_service.py index e4b6f3c..caeae06 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -129,6 +129,6 @@ def del_record_by_id(self, record_id): if not record: raise DNSError(DNSErrors.UNALLOWED, "This record does not exist.") - domain = self.domains.get_domain_by_id(record.domain_id) + domain = self.domains.get_domain_by_id(record.domain) self.records.del_record_by_id(record_id) self.ddns.del_record(domain.domain, record.type, record.value) From e89fa1f6d825ba520f9ce1f23874bbb55e7cb5f0 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 28 Nov 2023 02:45:42 -0500 Subject: [PATCH 22/93] debug: record ttl validation --- controllers/ddns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/ddns.py b/controllers/ddns.py index a46092a..f0ecd4b 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -57,11 +57,11 @@ def addRecord(domain, type_, value): try: req = request.json - if req and 'ttl' in req and req['ttl'].isnumeric() and 5 <= int(req['ttl']) <= 86400: + if req and 'ttl' in req and 5 <= int(req['ttl']) <= 86400: ttl = int(req['ttl']) else: ttl = 5 - except: + except Exception as e: ttl = 5 check_result = check_type(type_, value) From 8967a2a16e5d68e82df24f63233dc9d0532d84d3 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 28 Nov 2023 03:26:14 -0500 Subject: [PATCH 23/93] fix conflict --- controllers/domains.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/controllers/domains.py b/controllers/domains.py index 2476bb0..8dbd7cb 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -33,3 +33,19 @@ def release_domain(domain): return {"msg": "ok"} except Exception as e: return {"msg": e.msg}, 403 + +@app.route("/renew/", methods=['POST']) +def renew_domain(domain): + + if not g.user: + return {"message": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + try: + authService.authorize_action(g.user['uid'], Operation.RENEW, domain_name) + dnsService.renew_domain(domain_name) + return {"msg": "ok"} + except Exception as e: + return {"msg": e.msg}, 403 From bcafc402ed7ac08cd78d6a6bbb47165566e4f7d0 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 28 Nov 2023 05:06:32 -0500 Subject: [PATCH 24/93] Use msg in response uniformly --- controllers/auth.py | 2 +- controllers/ddns.py | 4 ++-- controllers/domains.py | 13 +++++++++---- services/auth_service.py | 4 +++- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/controllers/auth.py b/controllers/auth.py index f145541..1ef3b6e 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -32,4 +32,4 @@ def whoami(): data['email'] = g.user['email'] data['domains'] = dnsService.list_domains_by_user(g.user['uid']) return data - return {"message": "Unauth."}, 401 + return {"msg": "Unauth."}, 401 diff --git a/controllers/ddns.py b/controllers/ddns.py index f0ecd4b..69d807b 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -50,7 +50,7 @@ def check_type(type_, value): def addRecord(domain, type_, value): if not g.user: - return {"message": "Unauth."}, 401 + return {"msg": "Unauth."}, 401 domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) @@ -79,7 +79,7 @@ def addRecord(domain, type_, value): def delRecord(domain, type_, value): if not g.user: - return {"message": "Unauth."}, 401 + return {"msg": "Unauth."}, 401 domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) diff --git a/controllers/domains.py b/controllers/domains.py index 8dbd7cb..330390a 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -2,15 +2,20 @@ from main import app, authService, dnsService from services import Operation +import logging + @app.route("/domains/", methods=['POST']) def register_domain(domain): if not g.user: - return {"message": "Unauth."}, 401 + return {"msg": "Unauth."}, 401 - domain_struct = domain.lower().strip('/').split('/') + domain_struct = domain.replace('.', '/').lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) + if dnsService.check_domain(domain_name) != len(domain_struct): + return {"msg": "You can only register specific level domain name."}, 400 + try: authService.authorize_action(g.user['uid'], Operation.APPLY, domain_name) dnsService.register_domain(g.user['uid'], domain_name) @@ -22,7 +27,7 @@ def register_domain(domain): def release_domain(domain): if not g.user: - return {"message": "Unauth."}, 401 + return {"msg": "Unauth."}, 401 domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) @@ -38,7 +43,7 @@ def release_domain(domain): def renew_domain(domain): if not g.user: - return {"message": "Unauth."}, 401 + return {"msg": "Unauth."}, 401 domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) diff --git a/services/auth_service.py b/services/auth_service.py index 259b18d..619621c 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -34,12 +34,14 @@ def issue_token(self, profile): token['exp'] = (now) + 3600 token['iat'] = token['nbf'] = now token['uid'] = token['username'] + token['adm'] = False user = self.users.query(token['uid']) if user: if user.email != token['email']: self.users.update_email(token['uid'], token['email']) + token['isAdmin'] = user.isAdmin else: self.users.add(uid=token['uid'], name='', @@ -78,7 +80,7 @@ def authorize_action(self, uid, action, domain_name): raise UnauthorizedError("You cannot apply for more domains.") if action == Operation.RELEASE: domain = self.domains.get_domain(domain_name) - if domain.userId != uid: + if not domain or domain.userId != uid: raise UnauthorizedError("You cannot modify domain %s which you don't have." % (domain_name, )) if action == Operation.MODIFY: domain = self.domains.get_domain(domain_name) From 0d019327e081bde6330a3e22cef3a214c4792117 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 28 Nov 2023 05:06:32 -0500 Subject: [PATCH 25/93] Use msg in response uniformly, and update test-ddns testcase. --- tests/test_ddns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_ddns.py b/tests/test_ddns.py index 5eadea4..1ce6659 100644 --- a/tests/test_ddns.py +++ b/tests/test_ddns.py @@ -11,9 +11,9 @@ ], ) -testdata_A = [("test.nycu-dev.me", 'A', "140.113.89.64", 5), - ("test.nycu-dev.me", 'A', "140.113.64.89", 5), - ("test2.nycu-dev.me", 'A', "140.113.69.69", 86400), +testdata_A = [("test-ddns.nycu-dev.me", 'A', "140.113.89.64", 5), + ("test-ddns.nycu-dev.me", 'A', "140.113.64.89", 5), + ("test2-ddns.nycu-dev.me", 'A', "140.113.69.69", 86400), ] def test_add_A_record(): From 71bda4e6a7fc97a6e9ca3c2c5f1a5daf84968403 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 28 Nov 2023 13:07:11 +0000 Subject: [PATCH 26/93] reserved domain with short length --- controllers/domains.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/controllers/domains.py b/controllers/domains.py index 330390a..1354535 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -13,6 +13,9 @@ def register_domain(domain): domain_struct = domain.replace('.', '/').lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) + if len(domain_struct[-1]) < 4: + return {"msg": "Length must be greater than 3."}, 400 + if dnsService.check_domain(domain_name) != len(domain_struct): return {"msg": "You can only register specific level domain name."}, 400 From 69c42efbc1ee6138770d5aa761e0281c4425b576 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 28 Nov 2023 21:19:09 -0500 Subject: [PATCH 27/93] add support for ns record and add unittest for record ttl setting. --- controllers/ddns.py | 9 +++- .../{test_permission.py => test_attacker.py} | 14 +++++ tests/test_common.py | 3 +- tests/test_controllers.py | 51 +++++++++++++++++-- 4 files changed, 71 insertions(+), 6 deletions(-) rename tests/{test_permission.py => test_attacker.py} (61%) diff --git a/controllers/ddns.py b/controllers/ddns.py index 69d807b..224b1cb 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -5,7 +5,6 @@ from services import Operation - domainRegex = re.compile(r'^([A-Za-z0-9]\.|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]\.){1,3}[A-Za-z]{2,6}$') def is_ip(addr, protocol = ipaddress.IPv4Address): @@ -21,7 +20,9 @@ def is_domain(domain): return domainRegex.fullmatch(domain) def check_type(type_, value): - + + if type_ not in {'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS'}: + return {"errorType": "DNSError", "msg": f"Now allowed type {type_}."}, 403 if type_ == 'A': if not is_ip(value, ipaddress.IPv4Address): return {"errorType": "DNSError", "msg": "Type A with non-IPv4 value."}, 403 @@ -42,6 +43,10 @@ def check_type(type_, value): if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): return {"errorType": "DNSError", "msg": "Type TXT with value longer than 255 chars or more than 1 line."}, 403 + + if type_ == 'NS' and not is_domain(value): + return {"errorType": "DNSError", "msg": "Type NS with non-domain-name value."}, 403 + return None diff --git a/tests/test_permission.py b/tests/test_attacker.py similarity index 61% rename from tests/test_permission.py rename to tests/test_attacker.py index 488827b..94c3ce0 100644 --- a/tests/test_permission.py +++ b/tests/test_attacker.py @@ -10,22 +10,36 @@ def test_permission(): assert response.status_code == 200 response = requests.post(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h04) assert response.status_code == 403 + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-permission1") + assert response.status_code == 401 # Add record + response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64") + assert response.status_code == 401 response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h04) assert response.status_code == 403 response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h28) assert response.status_code == 200 # Delete record + response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64") + assert response.status_code == 401 response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h04) assert response.status_code == 403 response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h28) assert response.status_code == 200 # Release domain + response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-permission1") + assert response.status_code == 401 response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h04) assert response.status_code == 403 response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h28) assert response.status_code == 200 +def test_invalid_operation(): + headers = get_headers("109550032") + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-attack", headers = headers) + assert response.status_code == 200 + response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/TXT/attack message%0Ans2 A 103.179.29.12") + print(response.status_code) diff --git a/tests/test_common.py b/tests/test_common.py index 4def5f7..45120ef 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -10,4 +10,5 @@ def get_headers(uid): } response = requests.get(URL_BASE + "test_auth/", json = data) token = json.loads(response.text)['token'] - return {'Authorization': 'Bearer ' + token} \ No newline at end of file + headers = {'Authorization': f'Bearer {token}'} + return headers \ No newline at end of file diff --git a/tests/test_controllers.py b/tests/test_controllers.py index ffe0b56..da8d561 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -1,12 +1,10 @@ import json +import logging import time import requests import pydig from .test_common import get_headers, URL_BASE - - - resolver = pydig.Resolver( executable='/usr/bin/dig', nameservers=[ @@ -14,6 +12,29 @@ ], ) +def test_add_ttl(): + + headers = get_headers("109550032") + + def test_ttl(ttl, answer): + # Add record for ttl 10 + response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", json = {'ttl': ttl}, headers = headers) + assert response.status_code == 200 + response = requests.get(URL_BASE + "whoami/", headers = headers) + assert response.status_code == 200 + for domain in json.loads(response.text)['domains']: + if domain['domain'] == 'test-ttl1.nycu-dev.me': + assert domain['records'][0][3] == answer + response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", headers = headers) + + # Register domains + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-ttl1", headers = headers) + test_ttl(1, 5) + test_ttl(10, 10) + test_ttl(86401, 5) + test_ttl("random_string", 5) + response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-ttl1", headers = headers) + def test_register_and_release_domain(): headers = get_headers("109550004") # Register domains @@ -56,3 +77,27 @@ def test_add_and_delete_records(): assert response.status_code == 200 response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.64.89", headers = headers) assert response.status_code == 200 + +def test_add_ttl(): + + headers = get_headers("109550032") + + def test_ttl(ttl, answer): + # Add record for ttl 10 + response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", json = {'ttl': ttl}, headers = headers) + assert response.status_code == 200 + response = requests.get(URL_BASE + "whoami/", headers = headers) + assert response.status_code == 200 + for domain in json.loads(response.text)['domains']: + if domain['domain'] == 'test-ttl1.nycu-dev.me': + assert domain['records'][0][3] == answer + response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", headers = headers) + + # Register domains + response = requests.post(URL_BASE + "domains/me/nycu-dev/test-ttl1", headers = headers) + test_ttl(1, 5) + test_ttl(10, 10) + test_ttl(86401, 5) + test_ttl("random_string", 5) + response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-ttl1", headers = headers) + From 6afa7b10f967cfeef5d60f07e5eeddabe748666e Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Wed, 29 Nov 2023 02:51:37 +0000 Subject: [PATCH 28/93] recycling expired domains --- launch_thread.py | 38 ++++++++++++++++++++++++++++++++++++ main.py | 2 +- models/domains.py | 11 +++++++++++ tests/test_domain_expired.py | 38 ++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 launch_thread.py create mode 100644 tests/test_domain_expired.py diff --git a/launch_thread.py b/launch_thread.py new file mode 100644 index 0000000..d377ba3 --- /dev/null +++ b/launch_thread.py @@ -0,0 +1,38 @@ +import logging +import os +from sqlalchemy import create_engine + +import config +from models import Users, Domains, Records, DDNS, db +from services import DNSService + +env_test = os.getenv('TEST') + +sql_engine = None +if env_test is not None: + sql_engine = create_engine("sqlite:///:memory:") + db.Base.metadata.create_all(sql_engine) +else: + sql_engine = create_engine( + 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( + user=config.MYSQL_USER, + pswd=config.MYSQL_PSWD, + host=config.MYSQL_HOST, + db=config.MYSQL_DB + ) + ) + +ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) + +users = Users(sql_engine) +domains = Domains(sql_engine) +records = Records(sql_engine) + +authService = AuthService(logging, config.JWT_SECRET, users, domains) +dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) + +while True: + expired_domains = dnsService.get_expired_domains() + for domain in expired_domains: + dnsService.release_domain(domain.domain) + diff --git a/main.py b/main.py index 488999f..4aba067 100644 --- a/main.py +++ b/main.py @@ -8,7 +8,6 @@ from models import Users, Domains, Records, DDNS, db from services import AuthService, DNSService, Oauth - env_test = os.getenv('TEST') app = Flask(__name__) @@ -45,3 +44,4 @@ def index(): return "Hello World!" from controllers import auth, domains, ddns + diff --git a/models/domains.py b/models/domains.py index 4c111df..21cab68 100644 --- a/models/domains.py +++ b/models/domains.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import sessionmaker, scoped_session +from sqlalchemy import func from datetime import datetime, timedelta from . import db import logging @@ -16,6 +17,16 @@ def get_domain(self, domain_name): finally: session.close() + def get_expired_domain(self, domain_name): + session = self.make_session() + try: + now = func.now() + domains = session.query(db.Domain).filter_by(domain=domain_name, status=1).filter(db.Domain.expDate <= now).all() + + return domains + finally: + session.close() + def get_domain_by_id(self, domain_id): session = self.make_session() try: diff --git a/tests/test_domain_expired.py b/tests/test_domain_expired.py new file mode 100644 index 0000000..45d562f --- /dev/null +++ b/tests/test_domain_expired.py @@ -0,0 +1,38 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from datetime import datetime +from datetime import timedelta +import time +import logging + +from models import Domains, Records, Users, db, DDNS +from services import DNSService +import config + + +ddns = DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") + +sql_engine = create_engine('sqlite:///:memory:') +db.Base.metadata.create_all(sql_engine) +Session = sessionmaker(bind=sql_engine) +session = Session() + +users = Users(sql_engine) +domains = Domains(sql_engine) +records = Records(sql_engine) + +dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) + +def test_domain_expire(): + # Insert an expiring domain + exp_date = datetime.now() + timedelta(seconds=10) + domain = db.Domain(userId="109550028", + domain="test-expire.nycu-dev.me", + regDate=datetime.now(), + expDate=exp_date, + status=1) + session.add(domain) + session.commit() + # Waiting for expiring + time.sleep(15) + assert dnsService.get_domain("test-expire.nycu-dev.me") == None From 39ca881ec84e5bccdca1bdbcd1d93396f8dd210c Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 29 Nov 2023 00:05:35 -0500 Subject: [PATCH 29/93] automatically remove expired domains --- launch_thread.py | 17 +++++++++++------ models/domains.py | 18 ++++++++---------- services/dns_service.py | 3 +++ tests/test_domain_expired.py | 11 +++++++---- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/launch_thread.py b/launch_thread.py index d377ba3..0c34916 100644 --- a/launch_thread.py +++ b/launch_thread.py @@ -1,10 +1,16 @@ import logging import os +import time from sqlalchemy import create_engine import config from models import Users, Domains, Records, DDNS, db -from services import DNSService +from services import AuthService, DNSService + +def recycle(): + while (domain := dnsService.get_expired_domain()) != None: + logging.info(f"recycling {domain.domain}") + dnsService.release_domain(domain.domain) env_test = os.getenv('TEST') @@ -31,8 +37,7 @@ authService = AuthService(logging, config.JWT_SECRET, users, domains) dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) -while True: - expired_domains = dnsService.get_expired_domains() - for domain in expired_domains: - dnsService.release_domain(domain.domain) - +if __name__ == "__main__": + while True: + recycle() + time.sleep(1) diff --git a/models/domains.py b/models/domains.py index 21cab68..f2c3f77 100644 --- a/models/domains.py +++ b/models/domains.py @@ -1,5 +1,4 @@ from sqlalchemy.orm import sessionmaker, scoped_session -from sqlalchemy import func from datetime import datetime, timedelta from . import db import logging @@ -17,15 +16,14 @@ def get_domain(self, domain_name): finally: session.close() - def get_expired_domain(self, domain_name): - session = self.make_session() - try: - now = func.now() - domains = session.query(db.Domain).filter_by(domain=domain_name, status=1).filter(db.Domain.expDate <= now).all() - - return domains - finally: - session.close() + def get_expired_domain(self): + session = self.make_session() + try: + now = datetime.now() + domain = session.query(db.Domain).filter_by(status=1).filter(db.Domain.expDate < now).first() + return domain + finally: + session.close() def get_domain_by_id(self, domain_id): session = self.make_session() diff --git a/services/dns_service.py b/services/dns_service.py index caeae06..5e129fe 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -72,6 +72,9 @@ def get_domain(self, domain_name): record.ttl)) return domain_info + def get_expired_domain(self): + return self.domains.get_expired_domain() + def list_domains_by_user(self, uid): domains = [] for domain in self.domains.list_by_user(uid): diff --git a/tests/test_domain_expired.py b/tests/test_domain_expired.py index 45d562f..4e27584 100644 --- a/tests/test_domain_expired.py +++ b/tests/test_domain_expired.py @@ -9,10 +9,12 @@ from services import DNSService import config +from launch_thread import recycle + ddns = DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") -sql_engine = create_engine('sqlite:///:memory:') +sql_engine = create_engine("sqlite:///:memory:") db.Base.metadata.create_all(sql_engine) Session = sessionmaker(bind=sql_engine) session = Session() @@ -25,14 +27,15 @@ def test_domain_expire(): # Insert an expiring domain - exp_date = datetime.now() + timedelta(seconds=10) + exp_date = datetime.now() + timedelta(seconds=5) domain = db.Domain(userId="109550028", domain="test-expire.nycu-dev.me", regDate=datetime.now(), - expDate=exp_date, + expDate=datetime.now(), status=1) session.add(domain) session.commit() # Waiting for expiring - time.sleep(15) + time.sleep(10) + recycle() assert dnsService.get_domain("test-expire.nycu-dev.me") == None From 68b8f00ad0ad3fb82034857f6c3230b510f70961 Mon Sep 17 00:00:00 2001 From: roger Date: Wed, 29 Nov 2023 13:30:49 +0800 Subject: [PATCH 30/93] improve pylint score --- .pylintrc | 3 +- controllers/ddns.py | 68 +++++++++------- controllers/domains.py | 19 ++--- models/ddns.py | 27 +++---- models/domains.py | 11 ++- models/users.py | 22 +++--- services/auth_service.py | 40 +++++----- services/dns_service.py | 11 ++- services/nctu_oauth/oauth.py | 7 +- tests/test_attacker.py | 86 ++++++++++++++++---- tests/test_common.py | 6 +- tests/test_controllers.py | 147 ++++++++++++++++++++++++++++------- tests/test_ddns.py | 14 ++-- 13 files changed, 304 insertions(+), 157 deletions(-) diff --git a/.pylintrc b/.pylintrc index c7af0b8..e766285 100644 --- a/.pylintrc +++ b/.pylintrc @@ -142,7 +142,8 @@ disable=print-statement, duplicate-code, missing-function-docstring, missing-module-docstring, - missing-class-docstring + missing-class-docstring, + too-many-arguments # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/controllers/ddns.py b/controllers/ddns.py index 224b1cb..b138042 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -1,59 +1,76 @@ -from flask import Response, request, g -import re, ipaddress +import re +import ipaddress +from flask import request, g from main import app, authService, dnsService from services import Operation -domainRegex = re.compile(r'^([A-Za-z0-9]\.|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]\.){1,3}[A-Za-z]{2,6}$') +domain_regex = re.compile( + r'^([A-Za-z0-9]\.|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]\.){1,3}[A-Za-z]{2,6}$' +) -def is_ip(addr, protocol = ipaddress.IPv4Address): +def is_ip(addr, protocol=ipaddress.IPv4Address): try: ip = ipaddress.ip_address(addr) - if isinstance(ip, protocol): - return str(ip) - return False - except: + return str(ip) if isinstance(ip, protocol) else False + except Exception: return False def is_domain(domain): - return domainRegex.fullmatch(domain) + return domain_regex.fullmatch(domain) def check_type(type_, value): - + if type_ not in {'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS'}: - return {"errorType": "DNSError", "msg": f"Now allowed type {type_}."}, 403 + return { + "errorType": "DNSError", + "msg": f"Not allowed type {type_}." + }, 403 if type_ == 'A': if not is_ip(value, ipaddress.IPv4Address): - return {"errorType": "DNSError", "msg": "Type A with non-IPv4 value."}, 403 + return { + "errorType": "DNSError", + "msg": "Type A with non-IPv4 value." + }, 403 value = is_ip(value, ipaddress.IPv4Address) if type_ == 'AAAA': if not is_ip(value, ipaddress.IPv6Address): - return {"errorType": "DNSError", "msg": "Type AAAA with non-IPv6 value."}, 403 + return { + "errorType": "DNSError", + "msg": "Type AAAA with non-IPv6 value." + }, 403 value = is_ip(value, ipaddress.IPv6Address) if type_ == 'CNAME' and not is_domain(value): - return {"errorType": "DNSError", "msg": "Type CNAME with non-domain-name value."}, 403 + return { + "errorType": "DNSError", + "msg": "Type CNAME with non-domain-name value." + }, 403 if type_ == 'MX' and not is_domain(value): - return {"errorType": "DNSError", "msg": "Type MX with non-domain-name value."}, 403 + return { + "errorType": "DNSError", + "msg": "Type MX with non-domain-name value." + }, 403 if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): - return {"errorType": "DNSError", "msg": "Type TXT with value longer than 255 chars or more than 1 line."}, 403 - + return { + "errorType": "DNSError", + "msg": "Type TXT with value longer than 255 chars or more than 1 line." + }, 403 + + if type_ == 'NS' and not is_domain(value): return {"errorType": "DNSError", "msg": "Type NS with non-domain-name value."}, 403 - return None - @app.route("/ddns//records//", methods=['POST']) -def addRecord(domain, type_, value): - +def add_record(domain, type_, value): if not g.user: return {"msg": "Unauth."}, 401 @@ -66,7 +83,7 @@ def addRecord(domain, type_, value): ttl = int(req['ttl']) else: ttl = 5 - except Exception as e: + except Exception: ttl = 5 check_result = check_type(type_, value) @@ -78,11 +95,10 @@ def addRecord(domain, type_, value): dnsService.add_record(domain_name, type_, value, ttl) return {"msg": "ok"} except Exception as e: - return {"msg": e.msg}, 403 + return {"msg": str(e)}, 403 @app.route("/ddns//records//", methods=['DELETE']) -def delRecord(domain, type_, value): - +def del_record(domain, type_, value): if not g.user: return {"msg": "Unauth."}, 401 @@ -98,4 +114,4 @@ def delRecord(domain, type_, value): dnsService.del_record(domain_name, type_, value) return {"msg": "ok"} except Exception as e: - return {"msg": e.msg}, 403 + return {"msg": str(e)}, 403 diff --git a/controllers/domains.py b/controllers/domains.py index 1354535..29f5305 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -1,17 +1,14 @@ -from flask import Response, request, g +from flask import g from main import app, authService, dnsService from services import Operation -import logging - @app.route("/domains/", methods=['POST']) def register_domain(domain): - if not g.user: return {"msg": "Unauth."}, 401 domain_struct = domain.replace('.', '/').lower().strip('/').split('/') - domain_name = '.'.join(reversed(domain_struct)) + domain_name = '.'.join(reversed(domain_struct)) if len(domain_struct[-1]) < 4: return {"msg": "Length must be greater than 3."}, 400 @@ -24,36 +21,34 @@ def register_domain(domain): dnsService.register_domain(g.user['uid'], domain_name) return {"msg": "ok"} except Exception as e: - return {"msg": e.msg}, 403 + return {"msg": str(e)}, 403 @app.route("/domains/", methods=['DELETE']) def release_domain(domain): - if not g.user: return {"msg": "Unauth."}, 401 domain_struct = domain.lower().strip('/').split('/') - domain_name = '.'.join(reversed(domain_struct)) + domain_name = '.'.join(reversed(domain_struct)) try: authService.authorize_action(g.user['uid'], Operation.RELEASE, domain_name) dnsService.release_domain(domain_name) return {"msg": "ok"} except Exception as e: - return {"msg": e.msg}, 403 + return {"msg": str(e)}, 403 @app.route("/renew/", methods=['POST']) def renew_domain(domain): - if not g.user: return {"msg": "Unauth."}, 401 domain_struct = domain.lower().strip('/').split('/') - domain_name = '.'.join(reversed(domain_struct)) + domain_name = '.'.join(reversed(domain_struct)) try: authService.authorize_action(g.user['uid'], Operation.RENEW, domain_name) dnsService.renew_domain(domain_name) return {"msg": "ok"} except Exception as e: - return {"msg": e.msg}, 403 + return {"msg": str(e)}, 403 diff --git a/models/ddns.py b/models/ddns.py index 533a762..0c9fad2 100644 --- a/models/ddns.py +++ b/models/ddns.py @@ -9,13 +9,12 @@ class DDNS: def __launch(self): pr = subprocess.Popen( ['nsupdate', '-k', self.key_file], - bufsize = 0, - stdin = subprocess.PIPE, - stdout = subprocess.PIPE) + bufsize=0, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) if self.name_server: pr.stdin.write(f"server {self.name_server}\n".encode()) - if self.zone: pr.stdin.write(f"zone {self.zone}\n".encode()) @@ -27,22 +26,20 @@ def __write(self): try: while self.queue.qsize(): cmd = self.queue.get() + if self.nsupdate.poll() is not None: + self.nsupdate = self.__launch() + logging.warning("Subprocess nsupdate is dead, relaunched.") + self.nsupdate.stdin.write((cmd + "\n").encode()) diff = 1 - logging.debug("executing command: {cmd}".format(cmd=cmd)) - - if self.nsupdate.poll(): - self.queue.put(cmd) - self.nsupdate = self.__launch() - logging.warning("Subprocess nsupdate is dead.") + logging.debug("executing command: %s", cmd) - if diff and self.nsupdate.poll() == None: + if diff and self.nsupdate.poll() is None: diff = 0 self.nsupdate.stdin.write(b"send\n") except Exception as e: - logging.warning(e) - raise Exception(e) + logging.warning("Error in __write: %s", str(e)) time.sleep(5) @@ -61,10 +58,10 @@ def add_record(self, domain, rectype, value, ttl = 5): if domain != "" and rectype != "" and value != "": if rectype == "TXT": value = '"%s"' % value.replace('"', '\"') - self.queue.put("update add %s %d %s %s" % (domain, ttl, rectype, value)) + self.queue.put(f"update add {domain} {ttl} {rectype} {value}") def del_record(self, domain, rectype, value): if domain != "": if rectype == "TXT": value = '"%s"' % value.replace('"', '\"') - self.queue.put("update delete %s %s %s" % (domain, rectype, value)) + self.queue.put(f"update delete {domain} {rectype} {value}") diff --git a/models/domains.py b/models/domains.py index 4c111df..bc2e822 100644 --- a/models/domains.py +++ b/models/domains.py @@ -1,7 +1,7 @@ -from sqlalchemy.orm import sessionmaker, scoped_session from datetime import datetime, timedelta -from . import db import logging +from sqlalchemy.orm import sessionmaker, scoped_session +from . import db class Domains: def __init__(self, sql_engine): @@ -44,7 +44,7 @@ def register(self, domain_name, user_id): session.add(domain) session.commit() except Exception as e: - logging.error(f"Error registering domain: {e}") + logging.error("Error registering domain: %s", e) session.rollback() finally: session.close() @@ -57,7 +57,7 @@ def renew(self, domain_name): domain.expDate = datetime.now() + timedelta(days=30) session.commit() except Exception as e: - logging.error(f"Error renewing domain: {e}") + logging.error("Error renewing domain: %s", e) session.rollback() finally: session.close() @@ -71,8 +71,7 @@ def release(self, domain_name): domain.status = 0 session.commit() except Exception as e: - logging.error(f"Error releasing domain: {e}") + logging.error("Error releasing domain: %s", e) session.rollback() finally: session.close() - diff --git a/models/users.py b/models/users.py index 4011323..a94b73c 100644 --- a/models/users.py +++ b/models/users.py @@ -1,45 +1,45 @@ +import logging from sqlalchemy.orm import sessionmaker, scoped_session from . import db -import logging class Users: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) + self.session_factory = scoped_session(sessionmaker(bind=self.sql_engine)) def query(self, uid): - session = self.Session() + session = self.session_factory() try: user = session.query(db.User).filter_by(id=uid).first() return user except Exception as e: - logging.error(f"Error querying user: {e}") + logging.error("Error querying user: %s", e) return None finally: - session.close() + session.close() def add(self, uid, name, username, password, status, email): - session = self.Session() + session = self.session_factory() try: - user = db.User(id=uid, name=name, username=username, password=password, status=status, email=email) + user = db.User(id=uid, name=name, username=username, password=password, + status=status, email=email) session.add(user) session.commit() except Exception as e: - logging.error(f"Error adding user: {e}") + logging.error("Error adding user: %s", e) session.rollback() finally: session.close() def update_email(self, uid, email): - session = self.Session() + session = self.session_factory() try: user = session.query(db.User).filter_by(id=uid).first() if user: user.email = email session.commit() except Exception as e: - logging.error(f"Error updating user email: {e}") + logging.error("Error updating user email: %s", e) session.rollback() finally: session.close() - diff --git a/services/auth_service.py b/services/auth_service.py index 619621c..18025a1 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -1,14 +1,12 @@ -from sqlalchemy.orm import sessionmaker from datetime import timezone, datetime from enum import Enum import jwt - class Operation(Enum): - APPLY = 1 + APPLY = 1 RELEASE = 2 - MODIFY = 3 - RENEW = 4 + MODIFY = 3 + RENEW = 4 class UnauthorizedError(Exception): def __init__(self, msg): @@ -31,7 +29,7 @@ def issue_token(self, profile): now = int(datetime.now(tz=timezone.utc).timestamp()) token = profile token['iss'] = 'dns.nycu.me' - token['exp'] = (now) + 3600 + token['exp'] = now + 3600 token['iat'] = token['nbf'] = now token['uid'] = token['username'] token['adm'] = False @@ -43,12 +41,8 @@ def issue_token(self, profile): self.users.update_email(token['uid'], token['email']) token['isAdmin'] = user.isAdmin else: - self.users.add(uid=token['uid'], - name='', - username='', - password='', - status='active', - email=token['email']) + self.users.add(uid=token['uid'], name='', username='', password='', + status='active', email=token['email']) token = jwt.encode(token, self.jwt_secret, algorithm="HS256") return token @@ -60,19 +54,19 @@ def authenticate_token(self, payload): payload = payload.split(' ') if len(payload) != 2: raise UnauthorizedError("invalid payload") - tokenType, token = payload - if tokenType != 'Bearer': + token_type, token = payload + if token_type != 'Bearer': raise UnauthorizedError("invalid payload") try: payload = jwt.decode(token, self.jwt_secret, algorithms=["HS256"]) except Exception as e: - self.logger.debug(e.__str__()) + self.logger.debug(str(e)) return None return payload except UnauthorizedError as e: - self.logger.debug(e.__str__()) + self.logger.debug(str(e)) return None - + def authorize_action(self, uid, action, domain_name): if action == Operation.APPLY: domains = self.domains.list_by_user(uid) @@ -81,12 +75,18 @@ def authorize_action(self, uid, action, domain_name): if action == Operation.RELEASE: domain = self.domains.get_domain(domain_name) if not domain or domain.userId != uid: - raise UnauthorizedError("You cannot modify domain %s which you don't have." % (domain_name, )) + raise UnauthorizedError( + f"You cannot modify domain {domain_name} which you don't have." + ) if action == Operation.MODIFY: domain = self.domains.get_domain(domain_name) if domain.userId != uid: - raise UnauthorizedError("You cannot modify domain %s which you don't have." % (domain_name, )) + raise UnauthorizedError( + f"You cannot modify domain {domain_name} which you don't have." + ) if action == Operation.RENEW: domain = self.domains.get_domain(domain_name) if domain.userId != uid: - raise UnauthorizedError("You cannot modify domain %s which you don't have." % (domain_name, )) + raise UnauthorizedError( + f"You cannot modify domain {domain_name} which you don't have." + ) diff --git a/services/dns_service.py b/services/dns_service.py index caeae06..3614088 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -8,7 +8,7 @@ class DNSErrors(Enum): NXDOMAIN = "Non-ExistentDomain" DUPLICATED = "DuplicatedRecord" - UNALLOWED = "NotAllowedOperation" + UNALLOWED = "NotAllowedOperation" class DNSError(Exception): def __init__(self, typ, msg = ""): @@ -19,7 +19,7 @@ def __str__(self): return self.msg def __repr__(self): - return "%s: %s" % (self.typ, self.msg) + return f"{self.typ}: {self.msg}" class DNSService(): def __init__(self, logger, users, domains, records, ddns, host_domains): @@ -38,11 +38,10 @@ def check_domain(self, domain_name): return 0 def is_match(rule, struct): - # Check if the domain is matching to a specific rule rule = list(reversed(rule.split('.'))) if len(rule) > len(struct): return 0 - + for i in range(len(rule)): if rule[i] == '*': return i + 1 @@ -76,7 +75,7 @@ def list_domains_by_user(self, uid): domains = [] for domain in self.domains.list_by_user(uid): domain_info = self.get_domain(domain.domain) - domains.append(domain_info) + domains.append(domain_info) return domains def register_domain(self, uid, domain_name): @@ -114,7 +113,7 @@ def add_record(self, domain_name, type_, value, ttl): self.records.add_record(domain.id, type_, value, ttl) self.ddns.add_record(domain_name, type_, value, ttl) - + def del_record(self, domain_name, type_, value): domain_id = self.domains.get_domain(domain_name).id record = self.records.get_record_by_type_value(domain_id, diff --git a/services/nctu_oauth/oauth.py b/services/nctu_oauth/oauth.py index 42e9a0b..bd62d93 100644 --- a/services/nctu_oauth/oauth.py +++ b/services/nctu_oauth/oauth.py @@ -1,10 +1,9 @@ #-*- encoding: UTF-8 -*- - import requests OAUTH_URL = 'https://id.nycu.edu.tw' -class Oauth(object): +class Oauth: def __init__(self, redirect_uri, client_id, client_secret): self.grant_type = 'authorization_code' self.client_id = client_id @@ -21,7 +20,7 @@ def get_token(self, code): 'client_secret': self.client_secret, 'redirect_uri': self.redirect_uri } - result = requests.post(get_token_url, data=data) + result = requests.post(get_token_url, data=data, timeout=10) access_token = result.json().get('access_token', None) if access_token: @@ -36,6 +35,6 @@ def get_profile(self, token): } get_profile_url = OAUTH_URL + '/api/profile/' - data = requests.get(get_profile_url, headers=headers).json() + data = requests.get(get_profile_url, headers=headers, timeout=10).json() return data diff --git a/tests/test_attacker.py b/tests/test_attacker.py index 94c3ce0..2501b6b 100644 --- a/tests/test_attacker.py +++ b/tests/test_attacker.py @@ -4,42 +4,94 @@ def test_permission(): h04 = get_headers("109550004") h28 = get_headers("109550028") - + # Register domain - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h28) + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-permission1", + headers = h28, + timeout=10 + ) assert response.status_code == 200 - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h04) + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-permission1", + headers = h04, + timeout=10 + ) assert response.status_code == 403 - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-permission1") + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-permission1", + timeout=10 + ) assert response.status_code == 401 - + # Add record - response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64") + response = requests.post( + URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", + timeout=10 + ) assert response.status_code == 401 - response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h04) + response = requests.post( + URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", + headers = h04, + timeout=10 + ) assert response.status_code == 403 - response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h28) + response = requests.post( + URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", + headers = h28, + timeout=10 + ) assert response.status_code == 200 - + # Delete record - response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64") + response = requests.delete( + URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", + timeout=10 + ) assert response.status_code == 401 - response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h04) + response = requests.delete( + URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", + headers = h04, + timeout=10 + ) assert response.status_code == 403 - response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", headers = h28) + response = requests.delete( + URL_BASE + "ddns/me/nycu-dev/test-permission1/records/A/140.113.89.64", + headers = h28, + timeout=10 + ) assert response.status_code == 200 # Release domain - response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-permission1") + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-permission1", + timeout=10 + ) assert response.status_code == 401 - response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h04) + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-permission1", + headers = h04, + timeout=10 + ) assert response.status_code == 403 - response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-permission1", headers = h28) + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-permission1", + headers = h28, + timeout=10 + ) assert response.status_code == 200 def test_invalid_operation(): headers = get_headers("109550032") - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-attack", headers = headers) + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-attack", + headers = headers, + timeout=10 + ) assert response.status_code == 200 - response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-permission1/records/TXT/attack message%0Ans2 A 103.179.29.12") + response = requests.post( + URL_BASE + "ddns/me/nycu-dev/test-permission1/records/TXT/attack \ + message%0Ans2 A 103.179.29.12", + timeout=10 + ) print(response.status_code) diff --git a/tests/test_common.py b/tests/test_common.py index 45120ef..40ca5b8 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,5 +1,5 @@ -import requests import json +import requests URL_BASE = "http://172.21.21.4:8000/" @@ -8,7 +8,7 @@ def get_headers(uid): "email": "lin.cs09@nycu.edu.tw", "username": uid } - response = requests.get(URL_BASE + "test_auth/", json = data) + response = requests.get(URL_BASE + "test_auth/", json = data, timeout=10) token = json.loads(response.text)['token'] headers = {'Authorization': f'Bearer {token}'} - return headers \ No newline at end of file + return headers diff --git a/tests/test_controllers.py b/tests/test_controllers.py index da8d561..7731906 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -1,5 +1,4 @@ import json -import logging import time import requests import pydig @@ -13,91 +12,181 @@ ) def test_add_ttl(): - + headers = get_headers("109550032") - + def test_ttl(ttl, answer): # Add record for ttl 10 - response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", json = {'ttl': ttl}, headers = headers) + response = requests.post( + URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", + json = {'ttl': ttl}, + headers = headers, + timeout=10 + ) assert response.status_code == 200 - response = requests.get(URL_BASE + "whoami/", headers = headers) + response = requests.get( + URL_BASE + "whoami/", + headers = headers, + timeout=10 + ) assert response.status_code == 200 for domain in json.loads(response.text)['domains']: if domain['domain'] == 'test-ttl1.nycu-dev.me': assert domain['records'][0][3] == answer - response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", headers = headers) + response = requests.delete( + URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", + headers = headers, + timeout=10 + ) # Register domains - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-ttl1", headers = headers) + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-ttl1", + headers = headers, + timeout=10 + ) + test_ttl(1, 5) test_ttl(10, 10) test_ttl(86401, 5) test_ttl("random_string", 5) - response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-ttl1", headers = headers) + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-ttl1", + headers = headers, + timeout=10 + ) def test_register_and_release_domain(): headers = get_headers("109550004") # Register domains - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-route-reg1", headers = headers) + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-route-reg1", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-route-reg2", headers = headers) + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-route-reg2", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 # check if the entries exist - response = requests.get(URL_BASE + "/whoami/", headers = headers) + response = requests.get( + URL_BASE + "/whoami/", + headers = headers, + timeout=10 + ) domains = json.loads(response.text)['domains'] - assert {domain['domain'] for domain in domains} == {'test-route-reg1.nycu-dev.me', 'test-route-reg2.nycu-dev.me'} + assert {domain['domain'] for domain in domains} == {'test-route-reg1.nycu-dev.me', + 'test-route-reg2.nycu-dev.me'} # release the domains - response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-route-reg1", headers = headers) + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-route-reg1", + headers = headers, + timeout=10 + ) assert response.status_code == 200 - response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-route-reg2", headers = headers) + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-route-reg2", + headers = headers, + timeout=10 + ) assert response.status_code == 200 # check if the domain were released - response = requests.get(URL_BASE + "/whoami/", headers = headers) + response = requests.get( + URL_BASE + "/whoami/", + headers = headers, + timeout=10 + ) domains = json.loads(response.text)['domains'] assert {domain['domain'] for domain in domains} == set() def test_add_and_delete_records(): headers = get_headers("109550028") # Register domains - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-route-rec", headers = headers) + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-route-rec", + headers = headers, + timeout=10 + ) assert response.status_code == 200 # Add records - response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.89.64", headers = headers) + response = requests.post( + URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.89.64", + headers = headers, + timeout=10 + ) assert response.status_code == 200 - response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.64.89", headers = headers) + response = requests.post( + URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.64.89", + headers = headers, + timeout=10 + ) assert response.status_code == 200 # Check the result time.sleep(10) - assert set(resolver.query("test-route-rec.nycu-dev.me", 'A')) == {"140.113.89.64", "140.113.64.89"} + assert set(resolver.query("test-route-rec.nycu-dev.me", 'A')) == {"140.113.89.64", + "140.113.64.89"} # Remove the records - response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.89.64", headers = headers) + response = requests.delete( + URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.89.64", + headers = headers, + timeout=10 + ) assert response.status_code == 200 - response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.64.89", headers = headers) + response = requests.delete( + URL_BASE + "ddns/me/nycu-dev/test-route-rec/records/A/140.113.64.89", + headers = headers, + timeout=10 + ) assert response.status_code == 200 def test_add_ttl(): - + headers = get_headers("109550032") - + def test_ttl(ttl, answer): # Add record for ttl 10 - response = requests.post(URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", json = {'ttl': ttl}, headers = headers) + response = requests.post( + URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", + json = {'ttl': ttl}, + headers = headers, + timeout=10 + ) assert response.status_code == 200 - response = requests.get(URL_BASE + "whoami/", headers = headers) + response = requests.get( + URL_BASE + "whoami/", + headers = headers, + timeout=10 + ) assert response.status_code == 200 for domain in json.loads(response.text)['domains']: if domain['domain'] == 'test-ttl1.nycu-dev.me': assert domain['records'][0][3] == answer - response = requests.delete(URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", headers = headers) + response = requests.delete( + URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", + headers = headers, + timeout=10 + ) # Register domains - response = requests.post(URL_BASE + "domains/me/nycu-dev/test-ttl1", headers = headers) + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-ttl1", + headers = headers, + timeout=10 + ) test_ttl(1, 5) test_ttl(10, 10) test_ttl(86401, 5) test_ttl("random_string", 5) - response = requests.delete(URL_BASE + "domains/me/nycu-dev/test-ttl1", headers = headers) - + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-ttl1", + headers = headers, + timeout=10 + ) diff --git a/tests/test_ddns.py b/tests/test_ddns.py index 1ce6659..da51301 100644 --- a/tests/test_ddns.py +++ b/tests/test_ddns.py @@ -1,7 +1,7 @@ -import models.ddns import logging -import pydig import time +import pydig +import models.ddns ddns = models.ddns.DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") resolver = pydig.Resolver( @@ -19,15 +19,15 @@ def test_add_A_record(): domains = {} for testcase in testdata_A: - ddns.add_record(*testcase); + ddns.add_record(*testcase) if testcase[0] not in domains: domains[testcase[0]] = set() - domains[testcase[0]].add(testcase[2]); + domains[testcase[0]].add(testcase[2]) time.sleep(5) - for domain in domains: - assert set(resolver.query(domain, 'A')) == domains[domain] + for domain, expected_value in domains.items(): + assert set(resolver.query(domain, 'A')) == expected_value for testcase in testdata_A: ddns.del_record(*testcase[:-1]) time.sleep(5) for domain in domains: - assert resolver.query(domain, 'A') == [] + assert not resolver.query(domain, 'A') From ed2b53f1093a736d8915914b126281849c1c1f43 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Wed, 29 Nov 2023 06:35:06 +0000 Subject: [PATCH 31/93] add workflow: auto pull request to backend repo --- .github/workflows/pr_to_backend.yml | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/pr_to_backend.yml diff --git a/.github/workflows/pr_to_backend.yml b/.github/workflows/pr_to_backend.yml new file mode 100644 index 0000000..d193d5a --- /dev/null +++ b/.github/workflows/pr_to_backend.yml @@ -0,0 +1,32 @@ +name: Submodule updates to a parent repo + +on: + push: + branches: + - main + +jobs: + update: + name: Update submodules + runs-on: ubuntu-latest + environment: LinLee + steps: + - uses: actions/checkout@v3 + with: + repository: NYCU-ME/backend + token: ${{ secrets.TOKEN }} + + - name: Update submodules recursively + run: | + git submodule update --init --recursive + git submodule update --recursive --remote + + - name: Commit the changes + run: | + git config user.email "actions@github.com" + git config user.name "GitHub Actions - update submodules" + git add --all + git commit -m "Update submodules" || echo "No changes to commit" + git push + + From 7b03c55c35ce365b36a0b256c7e8828dad077c86 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 29 Nov 2023 07:04:49 -0500 Subject: [PATCH 32/93] add models and service for glue records --- launch_thread.py | 15 ++++++--- main.py | 7 ++-- models/__init__.py | 1 + models/db.py | 12 +++++++ models/ddns.py | 4 +-- models/glues.py | 63 +++++++++++++++++++++++++++++++++++ services/dns_service.py | 21 +++++++++++- tests/test_domain_expired.py | 5 +-- tests/test_domain_register.py | 15 +++++++-- 9 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 models/glues.py diff --git a/launch_thread.py b/launch_thread.py index 9d76409..f8ecbd9 100644 --- a/launch_thread.py +++ b/launch_thread.py @@ -4,13 +4,16 @@ from sqlalchemy import create_engine import config -from models import Users, Domains, Records, DDNS, db +from models import Users, Domains, Records, Glues, DDNS, db from services import AuthService, DNSService def recycle(dnsService): - while (domain := dnsService.get_expired_domain()) != None: - logging.info(f"recycling {domain.domain}") - dnsService.release_domain(domain.domain) + try: + while (domain := dnsService.get_expired_domain()) != None: + logging.info(f"recycling {domain.domain}") + dnsService.release_domain(domain.domain) + except Exception: + pass env_test = os.getenv('TEST') @@ -28,14 +31,16 @@ def recycle(dnsService): ) ) + ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) users = Users(sql_engine) domains = Domains(sql_engine) records = Records(sql_engine) +glues = Glues(sql_engine) authService = AuthService(logging, config.JWT_SECRET, users, domains) -dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) +dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) if __name__ == "__main__": while True: diff --git a/main.py b/main.py index 4aba067..a153bde 100644 --- a/main.py +++ b/main.py @@ -1,11 +1,12 @@ import logging import os +import time from flask import Flask import flask_cors from sqlalchemy import create_engine import config -from models import Users, Domains, Records, DDNS, db +from models import Users, Domains, Records, Glues, DDNS, db from services import AuthService, DNSService, Oauth env_test = os.getenv('TEST') @@ -32,12 +33,14 @@ users = Users(sql_engine) domains = Domains(sql_engine) records = Records(sql_engine) +glues = Glues(sql_engine) + nycu_oauth = Oauth(redirect_uri = config.NYCU_OAUTH_RURL, client_id = config.NYCU_OAUTH_ID, client_secret = config.NYCU_OAUTH_KEY) authService = AuthService(logging, config.JWT_SECRET, users, domains) -dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) +dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) @app.route("/") def index(): diff --git a/models/__init__.py b/models/__init__.py index f1d3393..b9e4edc 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -3,3 +3,4 @@ from .users import * from .domains import * from .records import * +from .glues import * diff --git a/models/db.py b/models/db.py index ea08367..c40c8e8 100644 --- a/models/db.py +++ b/models/db.py @@ -40,3 +40,15 @@ class Record(Base): regDate = Column(DateTime) expDate = Column(DateTime) status = Column(BOOLEAN, default=True) + +class Glue(Base): + __tablename__ = 'glues' + + id = Column(Integer, primary_key=True, autoincrement=True) + domain = Column(Integer, ForeignKey('domains.id'), nullable=False) + subdomain = Column(Text) + type = Column(CHAR(16), nullable=False) + value = Column(String(256)) + regDate = Column(DateTime) + expDate = Column(DateTime) + status = Column(BOOLEAN, default=True) diff --git a/models/ddns.py b/models/ddns.py index 0c9fad2..e4a164f 100644 --- a/models/ddns.py +++ b/models/ddns.py @@ -15,8 +15,8 @@ def __launch(self): if self.name_server: pr.stdin.write(f"server {self.name_server}\n".encode()) - if self.zone: - pr.stdin.write(f"zone {self.zone}\n".encode()) + #if self.zone: + # pr.stdin.write(f"zone {self.zone}\n".encode()) return pr diff --git a/models/glues.py b/models/glues.py new file mode 100644 index 0000000..e68dd04 --- /dev/null +++ b/models/glues.py @@ -0,0 +1,63 @@ +from datetime import datetime +from sqlalchemy.orm import sessionmaker, scoped_session + +from . import db + +class Glues: + def __init__(self, sql_engine): + self.sql_engine = sql_engine + self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) + + def get_record(self, glue_id): + session = self.Session() + try: + return session.query(db.Glue).filter_by(id=glue_id, status=1).first() + finally: + session.close() + + def get_records(self, domain_id): + session = self.Session() + try: + return session.query(db.Glue).filter_by(domain=domain_id, status=1).all() + finally: + session.close() + + def get_record_by_type_value(self, domain_id, subdomain, type_, value): + session = self.Session() + try: + return session.query(db.Glue).filter_by(domain=domain_id, + subdomain=subdomain, + type=type_, + value=value, + status=1).first() + finally: + session.close() + + def add_record(self, domain_id, subdomain, type_, value): + session = self.Session() + try: + new_record = db.Glue(domain=domain_id, subdomain=subdomain, type=type_, value=value, status=1, regDate=datetime.now()) + session.add(new_record) + session.commit() + return new_record + except Exception as e: + session.rollback() + raise e + finally: + session.close() + + def del_record(self, glue_id): + session = self.Session() + try: + record_to_delete = session.query(db.Glue).filter_by(id=glue_id).first() + if record_to_delete: + record_to_delete.expDate = datetime.now() + record_to_delete.status = 0 + session.commit() + return True + return False + except Exception as e: + session.rollback() + raise e + finally: + session.close() diff --git a/services/dns_service.py b/services/dns_service.py index d85c8ee..bff1209 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -22,11 +22,12 @@ def __repr__(self): return f"{self.typ}: {self.msg}" class DNSService(): - def __init__(self, logger, users, domains, records, ddns, host_domains): + def __init__(self, logger, users, domains, records, glues, ddns, host_domains): self.logger = logger self.users = users self.domains = domains self.records = records + self.glues = glues self.ddns = ddns self.host_domains = host_domains @@ -134,3 +135,21 @@ def del_record_by_id(self, record_id): domain = self.domains.get_domain_by_id(record.domain) self.records.del_record_by_id(record_id) self.ddns.del_record(domain.domain, record.type, record.value) + + def add_glue_record(self, domain_name, subdomain, type_, value): + domain = self.domains.get_domain(domain_name) + real_domain = f"{subdomain}.{domain_name}" + self.glues.add_record(domain.id, subdomain, type_, value) + self.ddns.add_record(real_domain, type_, value, 5) + + def del_glue_record(self, domain_name, subdomain, type_, value): + domain = self.domains.get_domain(domain_name) + real_domain = f"{subdomain}.{domain_name}" + glue_record = self.glues.get_record_by_type_value( + domain.id, + subdomain, + type_, + value + ) + self.glues.del_record(glue_record.id) + self.ddns.del_record(real_domain, type_, value) diff --git a/tests/test_domain_expired.py b/tests/test_domain_expired.py index bb64ec7..033cae8 100644 --- a/tests/test_domain_expired.py +++ b/tests/test_domain_expired.py @@ -5,7 +5,7 @@ import time import logging -from models import Domains, Records, Users, db, DDNS +from models import Domains, Records, Users, Glues, db, DDNS from services import DNSService import config @@ -22,8 +22,9 @@ users = Users(sql_engine) domains = Domains(sql_engine) records = Records(sql_engine) +glues = Glues(sql_engine) -dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) +dnsService = DNSService(logging, users, domains, glues, records, ddns, config.HOST_DOMAINS) def test_domain_expire(): # Insert an expiring domain diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index c59afe9..eca5f9b 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -4,7 +4,7 @@ import pydig import time -from models import Domains, Records, Users, db, DDNS +from models import Domains, Records, Users, Glues, db, DDNS from services import DNSService import config @@ -26,8 +26,9 @@ users = Users(sql_engine) domains = Domains(sql_engine) records = Records(sql_engine) +glues = Glues(sql_engine) -dnsService = DNSService(logging, users, domains, records, ddns, config.HOST_DOMAINS) +dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) testdata = [("test-reg.nycu-dev.me", 'A', "140.113.89.64", 5), ("test-reg.nycu-dev.me", 'A', "140.113.64.89", 5)] @@ -81,3 +82,13 @@ def test_duplicated_record(): except Exception: assert 1 dnsService.release_domain("test-add-dup-rec.nycu-dev.me") + +def test_glue_record(): + dnsService.register_domain("109550028", "test-glue.nycu-dev.me") + dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") + time.sleep(5) + assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == {"1.1.1.1"} + dnsService.del_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") + time.sleep(5) + assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() + dnsService.release_domain("test-glue.nycu-dev.me") From 341a0e6238ea580f38f211732aa2b1587cf93fa1 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 29 Nov 2023 07:31:35 -0500 Subject: [PATCH 33/93] add function: add and remove glue records --- controllers/__init__.py | 1 + controllers/glue.py | 108 ++++++++++++++++++++++++++++++++++ main.py | 3 +- services/dns_service.py | 10 ++++ tests/test_controllers.py | 32 ++++++++++ tests/test_domain_register.py | 7 +++ 6 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 controllers/glue.py diff --git a/controllers/__init__.py b/controllers/__init__.py index d1ebdc1..3330676 100644 --- a/controllers/__init__.py +++ b/controllers/__init__.py @@ -1,3 +1,4 @@ from .auth import * from .domains import * from .ddns import * +from .glue import * diff --git a/controllers/glue.py b/controllers/glue.py new file mode 100644 index 0000000..4b5d265 --- /dev/null +++ b/controllers/glue.py @@ -0,0 +1,108 @@ +import re +import ipaddress +from flask import request, g + +from main import app, authService, dnsService +from services import Operation + + +domain_regex = re.compile( + r'^([A-Za-z0-9]\.|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]\.){1,3}[A-Za-z]{2,6}$' +) + +def is_ip(addr, protocol=ipaddress.IPv4Address): + try: + ip = ipaddress.ip_address(addr) + return str(ip) if isinstance(ip, protocol) else False + except Exception: + return False + +def is_domain(domain): + return domain_regex.fullmatch(domain) + +def check_type(type_, value): + + if type_ not in {'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS'}: + return { + "errorType": "DNSError", + "msg": f"Not allowed type {type_}." + }, 403 + if type_ == 'A': + if not is_ip(value, ipaddress.IPv4Address): + return { + "errorType": "DNSError", + "msg": "Type A with non-IPv4 value." + }, 403 + + value = is_ip(value, ipaddress.IPv4Address) + + if type_ == 'AAAA': + if not is_ip(value, ipaddress.IPv6Address): + return { + "errorType": "DNSError", + "msg": "Type AAAA with non-IPv6 value." + }, 403 + + value = is_ip(value, ipaddress.IPv6Address) + + if type_ == 'CNAME' and not is_domain(value): + return { + "errorType": "DNSError", + "msg": "Type CNAME with non-domain-name value." + }, 403 + + if type_ == 'MX' and not is_domain(value): + return { + "errorType": "DNSError", + "msg": "Type MX with non-domain-name value." + }, 403 + + if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): + return { + "errorType": "DNSError", + "msg": "Type TXT with value longer than 255 chars or more than 1 line." + }, 403 + + + if type_ == 'NS' and not is_domain(value): + return {"errorType": "DNSError", "msg": "Type NS with non-domain-name value."}, 403 + + return None + +@app.route("/glue//records///", methods=['POST']) +def add_glue_record(domain, subdomain, type_, value): + if not g.user: + return {"msg": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + check_result = check_type(type_, value) + if check_result: + return check_result + + try: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + dnsService.add_glue_record(domain_name, subdomain, type_, value) + return {"msg": "ok"} + except Exception as e: + return {"msg": str(e)}, 403 + +@app.route("/glue//records///", methods=['DELETE']) +def del_glue_record(domain, subdomain, type_, value): + if not g.user: + return {"msg": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + check_result = check_type(type_, value) + if check_result: + return check_result + + try: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + dnsService.del_glue_record(domain_name, subdomain, type_, value) + return {"msg": "ok"} + except Exception as e: + return {"msg": str(e)}, 403 diff --git a/main.py b/main.py index a153bde..693570e 100644 --- a/main.py +++ b/main.py @@ -46,5 +46,4 @@ def index(): return "Hello World!" -from controllers import auth, domains, ddns - +from controllers import auth, domains, ddns, glue diff --git a/services/dns_service.py b/services/dns_service.py index bff1209..b482979 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -64,12 +64,19 @@ def get_domain(self, domain_name): domain_info['expDate'] = domain.expDate domain_info['domain'] = domain_name domain_info['records'] = [] + domain_info['glues'] = [] records = self.records.get_records(domain.id) + glues = self.glues.get_records(domain.id) for record in records: domain_info['records'].append((record.id, record.type, record.value, record.ttl)) + for record in glues: + glues.append((record.id, + record.subdomain, + record.type, + record.value)) return domain_info def get_expired_domain(self): @@ -101,8 +108,11 @@ def release_domain(self, domain_name): if not domain: raise DNSError(DNSErrors.NXDOMAIN, "This domain is not registered.") records = self.records.get_records(domain.id) + glues = self.glues.get_records(domain.id) for record in records: self.del_record_by_id(record.id) + for record in glues: + self.del_glue_record(domain.domain, record.subdomain, record.type, record.value) self.domains.release(domain_name) def add_record(self, domain_name, type_, value, ttl): diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 7731906..8fd26c2 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -147,6 +147,38 @@ def test_add_and_delete_records(): ) assert response.status_code == 200 +def test_auto_delete_glue_record(): + + headers = get_headers("110550029") + + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-glue-rec", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + + response = requests.post( + URL_BASE + "glue/me/nycu-dev/test-glue-rec/records/abc/A/1.1.1.1", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + + response = requests.delete( + URL_BASE + "glue/me/nycu-dev/test-glue-rec/records/abc/A/1.1.1.1", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-glue-rec", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + def test_add_ttl(): headers = get_headers("109550032") diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index eca5f9b..a2756c2 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -85,10 +85,17 @@ def test_duplicated_record(): def test_glue_record(): dnsService.register_domain("109550028", "test-glue.nycu-dev.me") + dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == {"1.1.1.1"} + dnsService.del_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() + + # check if glue record is be removed after domain released + dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") dnsService.release_domain("test-glue.nycu-dev.me") + time.sleep(5) + assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() From b0bee533c2137f74d8c2fa45339abe3e175b3c27 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 29 Nov 2023 10:01:21 -0500 Subject: [PATCH 34/93] add ttl for glue record --- controllers/glue.py | 11 ++++++++++- models/db.py | 1 + models/glues.py | 4 ++-- services/dns_service.py | 6 +++--- tests/test_domain_register.py | 4 ++-- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/controllers/glue.py b/controllers/glue.py index 4b5d265..052b8fa 100644 --- a/controllers/glue.py +++ b/controllers/glue.py @@ -77,13 +77,22 @@ def add_glue_record(domain, subdomain, type_, value): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) + try: + req = request.json + if req and 'ttl' in req and 5 <= int(req['ttl']) <= 86400: + ttl = int(req['ttl']) + else: + ttl = 5 + except Exception: + ttl = 5 + check_result = check_type(type_, value) if check_result: return check_result try: authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) - dnsService.add_glue_record(domain_name, subdomain, type_, value) + dnsService.add_glue_record(domain_name, subdomain, type_, value, ttl) return {"msg": "ok"} except Exception as e: return {"msg": str(e)}, 403 diff --git a/models/db.py b/models/db.py index c40c8e8..18952db 100644 --- a/models/db.py +++ b/models/db.py @@ -48,6 +48,7 @@ class Glue(Base): domain = Column(Integer, ForeignKey('domains.id'), nullable=False) subdomain = Column(Text) type = Column(CHAR(16), nullable=False) + ttl = Column(Integer, nullable=False) value = Column(String(256)) regDate = Column(DateTime) expDate = Column(DateTime) diff --git a/models/glues.py b/models/glues.py index e68dd04..4961d5a 100644 --- a/models/glues.py +++ b/models/glues.py @@ -33,10 +33,10 @@ def get_record_by_type_value(self, domain_id, subdomain, type_, value): finally: session.close() - def add_record(self, domain_id, subdomain, type_, value): + def add_record(self, domain_id, subdomain, type_, value, ttl): session = self.Session() try: - new_record = db.Glue(domain=domain_id, subdomain=subdomain, type=type_, value=value, status=1, regDate=datetime.now()) + new_record = db.Glue(domain=domain_id, subdomain=subdomain, type=type_, value=value, ttl=ttl, status=1, regDate=datetime.now()) session.add(new_record) session.commit() return new_record diff --git a/services/dns_service.py b/services/dns_service.py index b482979..51af08e 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -146,11 +146,11 @@ def del_record_by_id(self, record_id): self.records.del_record_by_id(record_id) self.ddns.del_record(domain.domain, record.type, record.value) - def add_glue_record(self, domain_name, subdomain, type_, value): + def add_glue_record(self, domain_name, subdomain, type_, value, ttl): domain = self.domains.get_domain(domain_name) real_domain = f"{subdomain}.{domain_name}" - self.glues.add_record(domain.id, subdomain, type_, value) - self.ddns.add_record(real_domain, type_, value, 5) + self.glues.add_record(domain.id, subdomain, type_, value, ttl) + self.ddns.add_record(real_domain, type_, value, ttl) def del_glue_record(self, domain_name, subdomain, type_, value): domain = self.domains.get_domain(domain_name) diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index a2756c2..76822d4 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -86,7 +86,7 @@ def test_duplicated_record(): def test_glue_record(): dnsService.register_domain("109550028", "test-glue.nycu-dev.me") - dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") + dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == {"1.1.1.1"} @@ -95,7 +95,7 @@ def test_glue_record(): assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() # check if glue record is be removed after domain released - dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") + dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) dnsService.release_domain("test-glue.nycu-dev.me") time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() From 61afa9262c03ebf1122e3f7a65ea0948417a2a01 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Wed, 29 Nov 2023 16:57:57 +0000 Subject: [PATCH 35/93] debug: fix typo --- services/dns_service.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/dns_service.py b/services/dns_service.py index 51af08e..28e5a84 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -73,10 +73,11 @@ def get_domain(self, domain_name): record.value, record.ttl)) for record in glues: - glues.append((record.id, - record.subdomain, - record.type, - record.value)) + domain_info['glues'].append((record.id, + record.subdomain, + record.type, + record.value, + record.ttl)) return domain_info def get_expired_domain(self): From 3b8fd61749141e1f7da3ee701d0d9a11ff21acd6 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Thu, 30 Nov 2023 01:21:35 -0500 Subject: [PATCH 36/93] add data statics --- config.py.sample | 5 +++++ controllers/domains.py | 26 ++++++++++++++++++++++++-- main.py | 3 ++- models/__init__.py | 1 + models/elastic.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 models/elastic.py diff --git a/config.py.sample b/config.py.sample index bf77d0c..4567e25 100644 --- a/config.py.sample +++ b/config.py.sample @@ -17,5 +17,10 @@ DDNS_KEY = r"/etc/ddnskey.conf" DDNS_SERVER = r"172.21.21.3" DDNS_ZONE = r"nycu-dev.me" +#ElasticSearch +ELASTICSERVER=172.21.21.7 +ELASTICUSER=elastic +ELASTICPASS=abc123 + #General HOST_DOMAINS = [r"*.nycu-dev.me"] diff --git a/controllers/domains.py b/controllers/domains.py index 29f5305..ec6c19a 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -1,5 +1,6 @@ from flask import g -from main import app, authService, dnsService +import datetime +from main import app, authService, dnsService, elastic from services import Operation @app.route("/domains/", methods=['POST']) @@ -30,7 +31,7 @@ def release_domain(domain): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) - + try: authService.authorize_action(g.user['uid'], Operation.RELEASE, domain_name) dnsService.release_domain(domain_name) @@ -52,3 +53,24 @@ def renew_domain(domain): return {"msg": "ok"} except Exception as e: return {"msg": str(e)}, 403 + +@app.route("/traffic/", methods=['GET']) +def get_domain_traffic(domain): + if not g.user: + return {"msg": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + data = [] + try: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + today = datetime.date.today() + for i in range(30, 0, -1): + past_date = today - datetime.timedelta(days=i) + date = past_date.strftime("%Y-%m-%d") + data.append(elastic.query_logs_count_by_data(domain_name, date)) + return {"msg": "ok", "data": data} + except Exception as e: + return {"msg": str(e)}, 403 + + diff --git a/main.py b/main.py index 693570e..e922460 100644 --- a/main.py +++ b/main.py @@ -6,7 +6,7 @@ from sqlalchemy import create_engine import config -from models import Users, Domains, Records, Glues, DDNS, db +from models import Users, Domains, Records, Glues, DDNS, Elastic, db from services import AuthService, DNSService, Oauth env_test = os.getenv('TEST') @@ -39,6 +39,7 @@ client_id = config.NYCU_OAUTH_ID, client_secret = config.NYCU_OAUTH_KEY) +elastic = Elastic(config.ELASTICSERVER, config.ELASTICUSER, config.ELASTICPASS) authService = AuthService(logging, config.JWT_SECRET, users, domains) dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) diff --git a/models/__init__.py b/models/__init__.py index b9e4edc..5fa50fb 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -4,3 +4,4 @@ from .domains import * from .records import * from .glues import * +from .elastic import * diff --git a/models/elastic.py b/models/elastic.py new file mode 100644 index 0000000..8eb7fba --- /dev/null +++ b/models/elastic.py @@ -0,0 +1,33 @@ +import requests +import json +import config + +class Elastic(): + + def __init__(self, server, user, password): + self.server = server + self.user = user + self.password = password + + def query_logs_count_by_date(self, domain, date): + url = f"http://{self.server}:9200/fluentd.named.dns/_search?pretty=true" + headers = {"Content-Type": "application/json"} + auth = (self.user, self.password) + data = { + "size": 0, + "query": { + "bool": { + "must": [ + {"query_string": {"query": f"*{domain}*"}}, + {"range": {"log_date": {"gte": date, "lte": date}}} + ] + } + } + } + + response = requests.get(url, headers=headers, auth=auth, data=json.dumps(data)) + response_json = response.json() + + return response_json.get('hits', {}).get('total', {}).get('value', 0) + + From a870fb1fb0d74ada16b01e5b6d0420c0ab6c023d Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Thu, 30 Nov 2023 09:18:23 -0500 Subject: [PATCH 37/93] introduce elasticsearch to calculate the traffic --- config.py.sample | 6 +++--- controllers/domains.py | 13 ++++++++----- models/elastic.py | 13 +++++++++---- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/config.py.sample b/config.py.sample index 4567e25..703b49b 100644 --- a/config.py.sample +++ b/config.py.sample @@ -18,9 +18,9 @@ DDNS_SERVER = r"172.21.21.3" DDNS_ZONE = r"nycu-dev.me" #ElasticSearch -ELASTICSERVER=172.21.21.7 -ELASTICUSER=elastic -ELASTICPASS=abc123 +ELASTICSERVER="172.21.21.7" +ELASTICUSER="elastic" +ELASTICPASS="abc123" #General HOST_DOMAINS = [r"*.nycu-dev.me"] diff --git a/controllers/domains.py b/controllers/domains.py index ec6c19a..2d26566 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -61,15 +61,18 @@ def get_domain_traffic(domain): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) - data = [] + + result = [] + today = datetime.date.today() + try: authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) - today = datetime.date.today() - for i in range(30, 0, -1): + for i in range(29, -1, -1): past_date = today - datetime.timedelta(days=i) date = past_date.strftime("%Y-%m-%d") - data.append(elastic.query_logs_count_by_data(domain_name, date)) - return {"msg": "ok", "data": data} + result.append((date, elastic.query(domain_name, date))) + + return {"msg": "ok", "data": result} except Exception as e: return {"msg": str(e)}, 403 diff --git a/models/elastic.py b/models/elastic.py index 8eb7fba..fc6c334 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -9,17 +9,22 @@ def __init__(self, server, user, password): self.user = user self.password = password - def query_logs_count_by_date(self, domain, date): + def query(self, domain, date): url = f"http://{self.server}:9200/fluentd.named.dns/_search?pretty=true" headers = {"Content-Type": "application/json"} auth = (self.user, self.password) + data = { - "size": 0, + "size": 0, "query": { "bool": { "must": [ - {"query_string": {"query": f"*{domain}*"}}, - {"range": {"log_date": {"gte": date, "lte": date}}} + {"match_phrase": {"log": f"({domain})"}}, + ], + "filter": [ + { + "term": {"log_time": date} + } ] } } From 3f4f86117c04f0b5b4e3b74378a8e7bfe0bf100d Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Fri, 1 Dec 2023 01:07:27 -0500 Subject: [PATCH 38/93] extend valid time from 30 days to 90 days --- models/domains.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/domains.py b/models/domains.py index 48dbc17..4e3c4eb 100644 --- a/models/domains.py +++ b/models/domains.py @@ -48,7 +48,7 @@ def register(self, domain_name, user_id): domain = db.Domain(userId=user_id, domain=domain_name, regDate=now, - expDate=now + timedelta(days=30), + expDate=now + timedelta(days=90), status=1) session.add(domain) session.commit() @@ -63,7 +63,7 @@ def renew(self, domain_name): try: domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() if domain: - domain.expDate = datetime.now() + timedelta(days=30) + domain.expDate = datetime.now() + timedelta(days=90) session.commit() except Exception as e: logging.error("Error renewing domain: %s", e) From bb0cf047233236c1cc07c2d0c2db84825ebb37b0 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Fri, 1 Dec 2023 02:37:41 -0500 Subject: [PATCH 39/93] Add administrator function --- controllers/domains.py | 23 +++++++++++++---- models/domains.py | 9 +++++++ services/auth_service.py | 11 +++++---- services/dns_service.py | 53 +++++++++++++++++++++++++--------------- 4 files changed, 66 insertions(+), 30 deletions(-) diff --git a/controllers/domains.py b/controllers/domains.py index 2d26566..f7fe6a8 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -3,6 +3,15 @@ from main import app, authService, dnsService, elastic from services import Operation +@app.route("/domains", methods=['GET']) +def list_domains(): + if not g.user: + return {"msg": "Unauth."}, 401 + if not g.user['isAdmin']: + return {"msg": "Unauth."}, 403 + + return {"msg": "ok", "data": dnsService.list_domains()} + @app.route("/domains/", methods=['POST']) def register_domain(domain): if not g.user: @@ -11,14 +20,15 @@ def register_domain(domain): domain_struct = domain.replace('.', '/').lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) - if len(domain_struct[-1]) < 4: + if not g.user['isAdmin'] and len(domain_struct[-1]) < 4: return {"msg": "Length must be greater than 3."}, 400 if dnsService.check_domain(domain_name) != len(domain_struct): return {"msg": "You can only register specific level domain name."}, 400 try: - authService.authorize_action(g.user['uid'], Operation.APPLY, domain_name) + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.APPLY, domain_name) dnsService.register_domain(g.user['uid'], domain_name) return {"msg": "ok"} except Exception as e: @@ -33,7 +43,8 @@ def release_domain(domain): domain_name = '.'.join(reversed(domain_struct)) try: - authService.authorize_action(g.user['uid'], Operation.RELEASE, domain_name) + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.RELEASE, domain_name) dnsService.release_domain(domain_name) return {"msg": "ok"} except Exception as e: @@ -48,7 +59,8 @@ def renew_domain(domain): domain_name = '.'.join(reversed(domain_struct)) try: - authService.authorize_action(g.user['uid'], Operation.RENEW, domain_name) + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.RENEW, domain_name) dnsService.renew_domain(domain_name) return {"msg": "ok"} except Exception as e: @@ -66,7 +78,8 @@ def get_domain_traffic(domain): today = datetime.date.today() try: - authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) for i in range(29, -1, -1): past_date = today - datetime.timedelta(days=i) date = past_date.strftime("%Y-%m-%d") diff --git a/models/domains.py b/models/domains.py index 4e3c4eb..7ec8f5c 100644 --- a/models/domains.py +++ b/models/domains.py @@ -41,6 +41,15 @@ def list_by_user(self, user_id): finally: session.close() + def list_all(self): + session = self.make_session() + try: + domains = session.query(db.Domain).filter_by(status=1).all() + return domains + finally: + session.close() + + def register(self, domain_name, user_id): session = self.make_session() try: diff --git a/services/auth_service.py b/services/auth_service.py index 18025a1..b0c4c54 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -32,7 +32,7 @@ def issue_token(self, profile): token['exp'] = now + 3600 token['iat'] = token['nbf'] = now token['uid'] = token['username'] - token['adm'] = False + token['isAdmin'] = False user = self.users.query(token['uid']) @@ -72,20 +72,21 @@ def authorize_action(self, uid, action, domain_name): domains = self.domains.list_by_user(uid) if len(domains) >= self.users.query(uid).limit: raise UnauthorizedError("You cannot apply for more domains.") + + domain = self.domains.get_domain(domain_name) if action == Operation.RELEASE: - domain = self.domains.get_domain(domain_name) - if not domain or domain.userId != uid: + if domain.userId != uid: raise UnauthorizedError( f"You cannot modify domain {domain_name} which you don't have." ) + if action == Operation.MODIFY: - domain = self.domains.get_domain(domain_name) if domain.userId != uid: raise UnauthorizedError( f"You cannot modify domain {domain_name} which you don't have." ) + if action == Operation.RENEW: - domain = self.domains.get_domain(domain_name) if domain.userId != uid: raise UnauthorizedError( f"You cannot modify domain {domain_name} which you don't have." diff --git a/services/dns_service.py b/services/dns_service.py index 28e5a84..d1a215c 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -31,6 +31,29 @@ def __init__(self, logger, users, domains, records, glues, ddns, host_domains): self.ddns = ddns self.host_domains = host_domains + def __get_domain_info(self, domain): + domain_info = {} + domain_info['id'] = domain.id + domain_info['regDate'] = domain.regDate + domain_info['expDate'] = domain.expDate + domain_info['domain'] = domain.domain + domain_info['records'] = [] + domain_info['glues'] = [] + records = self.records.get_records(domain.id) + glues = self.glues.get_records(domain.id) + for record in records: + domain_info['records'].append((record.id, + record.type, + record.value, + record.ttl)) + for record in glues: + domain_info['glues'].append((record.id, + record.subdomain, + record.type, + record.value, + record.ttl)) + return domain_info + def check_domain(self, domain_name): domain_struct = list(reversed(domain_name.split('.'))) @@ -58,26 +81,7 @@ def get_domain(self, domain_name): if not domain: return None - domain_info = {} - domain_info['id'] = domain.id - domain_info['regDate'] = domain.regDate - domain_info['expDate'] = domain.expDate - domain_info['domain'] = domain_name - domain_info['records'] = [] - domain_info['glues'] = [] - records = self.records.get_records(domain.id) - glues = self.glues.get_records(domain.id) - for record in records: - domain_info['records'].append((record.id, - record.type, - record.value, - record.ttl)) - for record in glues: - domain_info['glues'].append((record.id, - record.subdomain, - record.type, - record.value, - record.ttl)) + domain_info = self.__get_domain_info(domain) return domain_info def get_expired_domain(self): @@ -164,3 +168,12 @@ def del_glue_record(self, domain_name, subdomain, type_, value): ) self.glues.del_record(glue_record.id) self.ddns.del_record(real_domain, type_, value) + + def list_domains(self): + domains = self.domains.list_all() + result = [] + for domain in domains: + result.append(self.__get_domain_info(domain)) + return result + + From 3208c89e48ae29ed44069991420ef9489479c633 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sat, 2 Dec 2023 00:36:06 +0800 Subject: [PATCH 40/93] Update README.md --- README.md | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/README.md b/README.md index 94a0c6e..f66278b 100644 --- a/README.md +++ b/README.md @@ -1 +1,66 @@ # backend-flask-server + +## Architecture + +``` +. +├── config.py +├── config.py.sample +├── controllers +│   ├── auth.py +│   ├── ddns.py +│   ├── domains.py +│   ├── glue.py +│   └── __init__.py +├── launch_thread.py +├── main.py +├── models +│   ├── db.py +│   ├── ddns.py +│   ├── domains.py +│   ├── elastic.py +│   ├── glues.py +│   ├── __init__.py +│   ├── records.py +│   └── users.py +├── README.md +├── services +│   ├── auth_service.py +│   ├── dns_service.py +│   ├── __init__.py +│   └── nctu_oauth +│   ├── __init__.py +│   ├── oauth.py +│   └── README.md +└── tests + ├── __init__.py + ├── test_attacker.py + ├── test_common.py + ├── test_controllers.py + ├── test_ddns.py + ├── test_domain_expired.py + ├── test_domain_register.py + └── test_issue_token.py +``` + +### config.py 和 config.py.sample: +config.py 包含應用程序的配置信息，如數據庫連接設置、API金鑰等。 +config.py.sample 是 config.py 的範本，用來展示配置文件的格式和示例值。 + +### controllers: +這個目錄包含了應用程序的控制器或路由層，用於處理HTTP請求和路由它們到適當的功能模塊。 + +### launch_thread.py: +這個文件包含一個用於啟動回收 domain 的 process。 + +### main.py: +這個文件是應用程序的入口點，它可能包含主要的程式邏輯，如應用程序的初始化和啟動。# + +### models: +這個目錄包含應用程序的模型層，用於定義數據模型、數據庫表結構以及與數據庫的交互。 + +### services: +這個目錄包含應用程序的服務層，用於實現不同的業務邏輯，如身份驗證服務、DNS服務等。 + +### tests: +這個目錄包含測試用例，用於測試應用程序的各個部分，確保它們按照預期工作。 From b9e16c4282177316c1faef293b739da3d1793417 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sat, 2 Dec 2023 00:52:21 +0800 Subject: [PATCH 41/93] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f66278b..2c91df0 100644 --- a/README.md +++ b/README.md @@ -44,20 +44,20 @@ ``` ### config.py 和 config.py.sample: -config.py 包含應用程序的配置信息，如數據庫連接設置、API金鑰等。 +config.py 包含應用程序的配置信息，如資料庫連接設置、API金鑰等。 config.py.sample 是 config.py 的範本，用來展示配置文件的格式和示例值。 ### controllers: -這個目錄包含了應用程序的控制器或路由層，用於處理HTTP請求和路由它們到適當的功能模塊。 +這個目錄包含了應用程序的控制器或路由層，用於處理HTTP請求和路由它們到適當的功能模組。 ### launch_thread.py: 這個文件包含一個用於啟動回收 domain 的 process。 ### main.py: -這個文件是應用程序的入口點，它可能包含主要的程式邏輯，如應用程序的初始化和啟動。# +這個文件是應用程序的入口點，它可能包含主要的程式邏輯，如應用程序的初始化和啟動。 ### models: -這個目錄包含應用程序的模型層，用於定義數據模型、數據庫表結構以及與數據庫的交互。 +這個目錄包含應用程序的模型層，用於定義資料模型、資料庫表結構以及與資料庫的互動。 ### services: 這個目錄包含應用程序的服務層，用於實現不同的業務邏輯，如身份驗證服務、DNS服務等。 From b5e6fe7e12d13f1106f1636fbfecc9ecb7e9ec77 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sat, 2 Dec 2023 04:05:35 +0000 Subject: [PATCH 42/93] modify error message of invalid domain name and add subdomain check --- controllers/domains.py | 2 +- controllers/glue.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/controllers/domains.py b/controllers/domains.py index f7fe6a8..7cc3bf7 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -24,7 +24,7 @@ def register_domain(domain): return {"msg": "Length must be greater than 3."}, 400 if dnsService.check_domain(domain_name) != len(domain_struct): - return {"msg": "You can only register specific level domain name."}, 400 + return {"msg": "Not valid domain name."}, 400 try: if not g.user['isAdmin']: diff --git a/controllers/glue.py b/controllers/glue.py index 052b8fa..ee6040d 100644 --- a/controllers/glue.py +++ b/controllers/glue.py @@ -77,6 +77,9 @@ def add_glue_record(domain, subdomain, type_, value): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) + if dnsService.check_domain(f"{domain_name}.{subdomain}") != 1+len(domain_struct): + return {"msg": "Not valid subdomain."}, 400 + try: req = request.json if req and 'ttl' in req and 5 <= int(req['ttl']) <= 86400: From 796bc1b3c00ea5d08fee9735bdf95850564c83c9 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Fri, 1 Dec 2023 23:43:42 -0500 Subject: [PATCH 43/93] debug: cannot add gluerecord --- config.py.sample | 26 -------------------------- controllers/glue.py | 2 +- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 config.py.sample diff --git a/config.py.sample b/config.py.sample deleted file mode 100644 index 703b49b..0000000 --- a/config.py.sample +++ /dev/null @@ -1,26 +0,0 @@ -# Oauth Parameters -NYCU_OAUTH_ID = "" -NYCU_OAUTH_KEY = "" -NYCU_OAUTH_RURL = "" - -#MySQL Parameters -MYSQL_HOST = r"172.21.21.2" -MYSQL_USER = r"nycu_me" -MYSQL_PSWD = r"abc123" -MYSQL_DB = r"nycu_me_dns" - -#JWT Secret Key -JWT_SECRET = r"abc123" - -#DDNS -DDNS_KEY = r"/etc/ddnskey.conf" -DDNS_SERVER = r"172.21.21.3" -DDNS_ZONE = r"nycu-dev.me" - -#ElasticSearch -ELASTICSERVER="172.21.21.7" -ELASTICUSER="elastic" -ELASTICPASS="abc123" - -#General -HOST_DOMAINS = [r"*.nycu-dev.me"] diff --git a/controllers/glue.py b/controllers/glue.py index ee6040d..18d3bfe 100644 --- a/controllers/glue.py +++ b/controllers/glue.py @@ -77,7 +77,7 @@ def add_glue_record(domain, subdomain, type_, value): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) - if dnsService.check_domain(f"{domain_name}.{subdomain}") != 1+len(domain_struct): + if dnsService.check_domain(f"{domain_name}.{subdomain}"): return {"msg": "Not valid subdomain."}, 400 try: From f73e17c06b375727ea7a7b99051cdf31efb89570 Mon Sep 17 00:00:00 2001 From: roger Date: Sun, 3 Dec 2023 12:57:55 +0800 Subject: [PATCH 44/93] upgrade pylint score --- .pylintrc | 84 ++------------------------------- controllers/auth.py | 7 +-- controllers/ddns.py | 22 +++++---- controllers/domains.py | 18 ++++--- controllers/glue.py | 33 ++++++++----- launch_thread.py | 44 ++++++++--------- main.py | 32 +++++-------- models/ddns.py | 13 +++-- models/domains.py | 5 +- models/elastic.py | 13 +++-- models/glues.py | 21 ++++++--- models/records.py | 12 ++--- models/users.py | 2 +- services/dns_service.py | 29 ++++++------ services/nctu_oauth/__init__.py | 3 +- tests/test_controllers.py | 54 +++------------------ tests/test_ddns.py | 2 +- tests/test_domain_expired.py | 14 ++---- tests/test_domain_register.py | 8 ++-- tests/test_issue_token.py | 4 +- 20 files changed, 156 insertions(+), 264 deletions(-) diff --git a/.pylintrc b/.pylintrc index e766285..98f0c2b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -60,90 +60,12 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, - duplicate-code, +disable=duplicate-code, missing-function-docstring, missing-module-docstring, missing-class-docstring, - too-many-arguments + too-many-arguments, + too-few-public-methods # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/controllers/auth.py b/controllers/auth.py index 1ef3b6e..a45bcb7 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -1,6 +1,5 @@ from flask import request, g from main import env_test, app, nycu_oauth, authService, dnsService -import config @app.before_request def before_request(): @@ -13,16 +12,14 @@ def get_token(code): token = nycu_oauth.get_token(code) if token: return {'token': authService.issue_token(nycu_oauth.get_profile(token))} - else: - return {'msg': "Invalid code."}, 401 + return {'msg': "Invalid code."}, 401 @app.route("/test_auth/", methods = ['GET']) def get_token_for_test(): if env_test: return {'msg': 'ok', 'token': authService.issue_token(request.json)} - else: - return {'msg': "It is not currently running on testing mode."}, 401 + return {'msg': "It is not currently running on testing mode."}, 401 @app.route("/whoami/", methods = ['GET']) def whoami(): diff --git a/controllers/ddns.py b/controllers/ddns.py index b138042..acd044e 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -22,14 +22,17 @@ def is_domain(domain): def check_type(type_, value): + error_response = None + if type_ not in {'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS'}: - return { + error_response = { "errorType": "DNSError", "msg": f"Not allowed type {type_}." }, 403 + if type_ == 'A': if not is_ip(value, ipaddress.IPv4Address): - return { + error_response = { "errorType": "DNSError", "msg": "Type A with non-IPv4 value." }, 403 @@ -38,7 +41,7 @@ def check_type(type_, value): if type_ == 'AAAA': if not is_ip(value, ipaddress.IPv6Address): - return { + error_response = { "errorType": "DNSError", "msg": "Type AAAA with non-IPv6 value." }, 403 @@ -46,28 +49,31 @@ def check_type(type_, value): value = is_ip(value, ipaddress.IPv6Address) if type_ == 'CNAME' and not is_domain(value): - return { + error_response = { "errorType": "DNSError", "msg": "Type CNAME with non-domain-name value." }, 403 if type_ == 'MX' and not is_domain(value): - return { + error_response = { "errorType": "DNSError", "msg": "Type MX with non-domain-name value." }, 403 if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): - return { + error_response = { "errorType": "DNSError", "msg": "Type TXT with value longer than 255 chars or more than 1 line." }, 403 if type_ == 'NS' and not is_domain(value): - return {"errorType": "DNSError", "msg": "Type NS with non-domain-name value."}, 403 + error_response = { + "errorType": "DNSError", + "msg": "Type NS with non-domain-name value." + }, 403 - return None + return error_response @app.route("/ddns//records//", methods=['POST']) def add_record(domain, type_, value): diff --git a/controllers/domains.py b/controllers/domains.py index 7cc3bf7..93dfb5f 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -1,5 +1,5 @@ -from flask import g import datetime +from flask import g from main import app, authService, dnsService, elastic from services import Operation @@ -7,9 +7,9 @@ def list_domains(): if not g.user: return {"msg": "Unauth."}, 401 - if not g.user['isAdmin']: + if not g.user['isAdmin']: return {"msg": "Unauth."}, 403 - + return {"msg": "ok", "data": dnsService.list_domains()} @app.route("/domains/", methods=['POST']) @@ -27,7 +27,7 @@ def register_domain(domain): return {"msg": "Not valid domain name."}, 400 try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.APPLY, domain_name) dnsService.register_domain(g.user['uid'], domain_name) return {"msg": "ok"} @@ -41,9 +41,9 @@ def release_domain(domain): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) - + try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.RELEASE, domain_name) dnsService.release_domain(domain_name) return {"msg": "ok"} @@ -59,7 +59,7 @@ def renew_domain(domain): domain_name = '.'.join(reversed(domain_struct)) try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.RENEW, domain_name) dnsService.renew_domain(domain_name) return {"msg": "ok"} @@ -78,7 +78,7 @@ def get_domain_traffic(domain): today = datetime.date.today() try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) for i in range(29, -1, -1): past_date = today - datetime.timedelta(days=i) @@ -88,5 +88,3 @@ def get_domain_traffic(domain): return {"msg": "ok", "data": result} except Exception as e: return {"msg": str(e)}, 403 - - diff --git a/controllers/glue.py b/controllers/glue.py index 18d3bfe..8bb71c6 100644 --- a/controllers/glue.py +++ b/controllers/glue.py @@ -5,7 +5,6 @@ from main import app, authService, dnsService from services import Operation - domain_regex = re.compile( r'^([A-Za-z0-9]\.|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]\.){1,3}[A-Za-z]{2,6}$' ) @@ -22,14 +21,17 @@ def is_domain(domain): def check_type(type_, value): + error_response = None + if type_ not in {'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS'}: - return { + error_response = { "errorType": "DNSError", "msg": f"Not allowed type {type_}." }, 403 + if type_ == 'A': if not is_ip(value, ipaddress.IPv4Address): - return { + error_response = { "errorType": "DNSError", "msg": "Type A with non-IPv4 value." }, 403 @@ -38,7 +40,7 @@ def check_type(type_, value): if type_ == 'AAAA': if not is_ip(value, ipaddress.IPv6Address): - return { + error_response = { "errorType": "DNSError", "msg": "Type AAAA with non-IPv6 value." }, 403 @@ -46,30 +48,36 @@ def check_type(type_, value): value = is_ip(value, ipaddress.IPv6Address) if type_ == 'CNAME' and not is_domain(value): - return { + error_response = { "errorType": "DNSError", "msg": "Type CNAME with non-domain-name value." }, 403 if type_ == 'MX' and not is_domain(value): - return { + error_response = { "errorType": "DNSError", "msg": "Type MX with non-domain-name value." }, 403 if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): - return { + error_response = { "errorType": "DNSError", "msg": "Type TXT with value longer than 255 chars or more than 1 line." }, 403 if type_ == 'NS' and not is_domain(value): - return {"errorType": "DNSError", "msg": "Type NS with non-domain-name value."}, 403 + error_response = { + "errorType": "DNSError", + "msg": "Type NS with non-domain-name value." + }, 403 - return None + return error_response -@app.route("/glue//records///", methods=['POST']) +@app.route( + "/glue//records///", + methods=['POST'] +) def add_glue_record(domain, subdomain, type_, value): if not g.user: return {"msg": "Unauth."}, 401 @@ -100,7 +108,10 @@ def add_glue_record(domain, subdomain, type_, value): except Exception as e: return {"msg": str(e)}, 403 -@app.route("/glue//records///", methods=['DELETE']) +@app.route( + "/glue//records///", + methods=['DELETE'] +) def del_glue_record(domain, subdomain, type_, value): if not g.user: return {"msg": "Unauth."}, 401 diff --git a/launch_thread.py b/launch_thread.py index f8ecbd9..a6f1456 100644 --- a/launch_thread.py +++ b/launch_thread.py @@ -7,42 +7,38 @@ from models import Users, Domains, Records, Glues, DDNS, db from services import AuthService, DNSService -def recycle(dnsService): +def recycle(local_dns_service): try: - while (domain := dnsService.get_expired_domain()) != None: - logging.info(f"recycling {domain.domain}") - dnsService.release_domain(domain.domain) + while (domain := local_dns_service.get_expired_domain()) is not None: + logging.info("recycling %s", domain.domain) + local_dns_service.release_domain(domain.domain) except Exception: pass -env_test = os.getenv('TEST') +ENV_TEST = os.getenv('TEST') -sql_engine = None -if env_test is not None: - sql_engine = create_engine("sqlite:///:memory:") - db.Base.metadata.create_all(sql_engine) +SQL_ENGINE = None +if ENV_TEST is not None: + SQL_ENGINE = create_engine("sqlite:///:memory:") + db.Base.metadata.create_all(SQL_ENGINE) else: - sql_engine = create_engine( - 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( - user=config.MYSQL_USER, - pswd=config.MYSQL_PSWD, - host=config.MYSQL_HOST, - db=config.MYSQL_DB - ) + connection_string = ( + f"mysql+pymysql://{config.MYSQL_USER}:{config.MYSQL_PSWD}" + f"@{config.MYSQL_HOST}/{config.MYSQL_DB}" ) - + SQL_ENGINE = create_engine(connection_string) ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) -users = Users(sql_engine) -domains = Domains(sql_engine) -records = Records(sql_engine) -glues = Glues(sql_engine) +users = Users(SQL_ENGINE) +domains = Domains(SQL_ENGINE) +records = Records(SQL_ENGINE) +glues = Glues(SQL_ENGINE) -authService = AuthService(logging, config.JWT_SECRET, users, domains) -dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) +auth_service = AuthService(logging, config.JWT_SECRET, users, domains) +dns_service = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) if __name__ == "__main__": while True: - recycle(dnsService) + recycle(dns_service) time.sleep(1) diff --git a/main.py b/main.py index e922460..6c491af 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,5 @@ import logging import os -import time from flask import Flask import flask_cors from sqlalchemy import create_engine @@ -14,26 +13,23 @@ app = Flask(__name__) flask_cors.CORS(app) -sql_engine = None +SQL_ENGINE = None if env_test is not None: - sql_engine = create_engine("sqlite:///:memory:") - db.Base.metadata.create_all(sql_engine) + SQL_ENGINE = create_engine("sqlite:///:memory:") + db.Base.metadata.create_all(SQL_ENGINE) else: - sql_engine = create_engine( - 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( - user=config.MYSQL_USER, - pswd=config.MYSQL_PSWD, - host=config.MYSQL_HOST, - db=config.MYSQL_DB - ) + connection_string = ( + f"mysql+pymysql://{config.MYSQL_USER}:{config.MYSQL_PSWD}" + f"@{config.MYSQL_HOST}/{config.MYSQL_DB}" ) + sql_engine = create_engine(connection_string) ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) -users = Users(sql_engine) -domains = Domains(sql_engine) -records = Records(sql_engine) -glues = Glues(sql_engine) +users = Users(SQL_ENGINE) +domains = Domains(SQL_ENGINE) +records = Records(SQL_ENGINE) +glues = Glues(SQL_ENGINE) nycu_oauth = Oauth(redirect_uri = config.NYCU_OAUTH_RURL, client_id = config.NYCU_OAUTH_ID, @@ -43,8 +39,4 @@ authService = AuthService(logging, config.JWT_SECRET, users, domains) dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) -@app.route("/") -def index(): - return "Hello World!" - -from controllers import auth, domains, ddns, glue +from controllers import auth, domains, ddns, glue # pylint: disable=all diff --git a/models/ddns.py b/models/ddns.py index e4a164f..5bd86cb 100644 --- a/models/ddns.py +++ b/models/ddns.py @@ -6,12 +6,13 @@ class DDNS: - def __launch(self): - pr = subprocess.Popen( + def __launch(self): + pr = subprocess.Popen( # pylint: disable=all ['nsupdate', '-k', self.key_file], bufsize=0, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + stdout=subprocess.PIPE + ) if self.name_server: pr.stdin.write(f"server {self.name_server}\n".encode()) @@ -57,11 +58,13 @@ def __init__(self, logger, key_file, name_server, zone): def add_record(self, domain, rectype, value, ttl = 5): if domain != "" and rectype != "" and value != "": if rectype == "TXT": - value = '"%s"' % value.replace('"', '\"') + value = value.replace('"', '\"') + value = f'"{value}"' self.queue.put(f"update add {domain} {ttl} {rectype} {value}") def del_record(self, domain, rectype, value): if domain != "": if rectype == "TXT": - value = '"%s"' % value.replace('"', '\"') + value = value.replace('"', '\"') + value = f'"{value}"' self.queue.put(f"update delete {domain} {rectype} {value}") diff --git a/models/domains.py b/models/domains.py index 7ec8f5c..75c6095 100644 --- a/models/domains.py +++ b/models/domains.py @@ -20,7 +20,10 @@ def get_expired_domain(self): session = self.make_session() try: now = datetime.now() - domain = session.query(db.Domain).filter_by(status=1).filter(db.Domain.expDate < now).first() + domain = session.query(db.Domain)\ + .filter_by(status=1)\ + .filter(db.Domain.expDate < now)\ + .first() return domain finally: session.close() diff --git a/models/elastic.py b/models/elastic.py index fc6c334..3d32b37 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -1,6 +1,5 @@ -import requests import json -import config +import requests class Elastic(): @@ -30,9 +29,13 @@ def query(self, domain, date): } } - response = requests.get(url, headers=headers, auth=auth, data=json.dumps(data)) + response = requests.get( + url, + headers=headers, + auth=auth, + data=json.dumps(data), + timeout=3 + ) response_json = response.json() return response_json.get('hits', {}).get('total', {}).get('value', 0) - - diff --git a/models/glues.py b/models/glues.py index 4961d5a..5227460 100644 --- a/models/glues.py +++ b/models/glues.py @@ -6,24 +6,24 @@ class Glues: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) + self.session = scoped_session(sessionmaker(bind=self.sql_engine)) def get_record(self, glue_id): - session = self.Session() + session = self.session() try: return session.query(db.Glue).filter_by(id=glue_id, status=1).first() finally: session.close() def get_records(self, domain_id): - session = self.Session() + session = self.session() try: return session.query(db.Glue).filter_by(domain=domain_id, status=1).all() finally: session.close() def get_record_by_type_value(self, domain_id, subdomain, type_, value): - session = self.Session() + session = self.session() try: return session.query(db.Glue).filter_by(domain=domain_id, subdomain=subdomain, @@ -34,9 +34,16 @@ def get_record_by_type_value(self, domain_id, subdomain, type_, value): session.close() def add_record(self, domain_id, subdomain, type_, value, ttl): - session = self.Session() + session = self.session() try: - new_record = db.Glue(domain=domain_id, subdomain=subdomain, type=type_, value=value, ttl=ttl, status=1, regDate=datetime.now()) + new_record = db.Glue( + domain=domain_id, + subdomain=subdomain, + type=type_, value=value, + ttl=ttl, + status=1, + regDate=datetime.now() + ) session.add(new_record) session.commit() return new_record @@ -47,7 +54,7 @@ def add_record(self, domain_id, subdomain, type_, value, ttl): session.close() def del_record(self, glue_id): - session = self.Session() + session = self.session() try: record_to_delete = session.query(db.Glue).filter_by(id=glue_id).first() if record_to_delete: diff --git a/models/records.py b/models/records.py index d45064e..44f08a2 100644 --- a/models/records.py +++ b/models/records.py @@ -6,24 +6,24 @@ class Records: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) + self.session_maker = scoped_session(sessionmaker(bind=self.sql_engine)) def get_record(self, record_id): - session = self.Session() + session = self.session_maker() try: return session.query(db.Record).filter_by(id=record_id, status=1).first() finally: session.close() def get_records(self, domain_id): - session = self.Session() + session = self.session_maker() try: return session.query(db.Record).filter_by(domain=domain_id, status=1).all() finally: session.close() def get_record_by_type_value(self, domain_id, type_, value): - session = self.Session() + session = self.session_maker() try: return session.query(db.Record).filter_by(domain=domain_id, type=type_, @@ -33,7 +33,7 @@ def get_record_by_type_value(self, domain_id, type_, value): session.close() def add_record(self, domain_id, record_type, value, ttl): - session = self.Session() + session = self.session_maker() try: record = db.Record(domain=domain_id, type=record_type, @@ -50,7 +50,7 @@ def add_record(self, domain_id, record_type, value, ttl): session.close() def del_record_by_id(self, record_id): - session = self.Session() + session = self.session_maker() try: record = session.query(db.Record).filter_by(id=record_id).first() if record: diff --git a/models/users.py b/models/users.py index a94b73c..d642dd2 100644 --- a/models/users.py +++ b/models/users.py @@ -21,7 +21,7 @@ def query(self, uid): def add(self, uid, name, username, password, status, email): session = self.session_factory() try: - user = db.User(id=uid, name=name, username=username, password=password, + user = db.User(id=uid, name=name, username=username, password=password, status=status, email=email) session.add(user) session.commit() diff --git a/services/dns_service.py b/services/dns_service.py index d1a215c..6c05e85 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -1,17 +1,15 @@ -import time import re from enum import Enum - DOMAIN_REGEX = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(? len(struct): return 0 - for i in range(len(rule)): - if rule[i] == '*': + for i, element in enumerate(rule): + if element == '*': return i + 1 - elif rule[i] != struct[i]: + if element != struct[i]: return 0 + return None + for domain in self.host_domains: - if (x:=is_match(domain, domain_struct)): - return x + match_result = is_match(domain, domain_struct) + if match_result is not None: + return match_result + + return None def get_domain(self, domain_name): domain = self.domains.get_domain(domain_name) @@ -168,12 +171,10 @@ def del_glue_record(self, domain_name, subdomain, type_, value): ) self.glues.del_record(glue_record.id) self.ddns.del_record(real_domain, type_, value) - + def list_domains(self): domains = self.domains.list_all() result = [] for domain in domains: result.append(self.__get_domain_info(domain)) return result - - diff --git a/services/nctu_oauth/__init__.py b/services/nctu_oauth/__init__.py index 465ec8f..dc86616 100644 --- a/services/nctu_oauth/__init__.py +++ b/services/nctu_oauth/__init__.py @@ -1,3 +1,2 @@ #-*- encoding: UTF-8 -*- - -from .oauth import Oauth \ No newline at end of file +from .oauth import Oauth diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 8fd26c2..4877c11 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -11,51 +11,6 @@ ], ) -def test_add_ttl(): - - headers = get_headers("109550032") - - def test_ttl(ttl, answer): - # Add record for ttl 10 - response = requests.post( - URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", - json = {'ttl': ttl}, - headers = headers, - timeout=10 - ) - assert response.status_code == 200 - response = requests.get( - URL_BASE + "whoami/", - headers = headers, - timeout=10 - ) - assert response.status_code == 200 - for domain in json.loads(response.text)['domains']: - if domain['domain'] == 'test-ttl1.nycu-dev.me': - assert domain['records'][0][3] == answer - response = requests.delete( - URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", - headers = headers, - timeout=10 - ) - - # Register domains - response = requests.post( - URL_BASE + "domains/me/nycu-dev/test-ttl1", - headers = headers, - timeout=10 - ) - - test_ttl(1, 5) - test_ttl(10, 10) - test_ttl(86401, 5) - test_ttl("random_string", 5) - response = requests.delete( - URL_BASE + "domains/me/nycu-dev/test-ttl1", - headers = headers, - timeout=10 - ) - def test_register_and_release_domain(): headers = get_headers("109550004") # Register domains @@ -148,7 +103,7 @@ def test_add_and_delete_records(): assert response.status_code == 200 def test_auto_delete_glue_record(): - + headers = get_headers("110550029") response = requests.post( @@ -157,7 +112,7 @@ def test_auto_delete_glue_record(): timeout=10 ) assert response.status_code == 200 - + response = requests.post( URL_BASE + "glue/me/nycu-dev/test-glue-rec/records/abc/A/1.1.1.1", headers = headers, @@ -171,7 +126,7 @@ def test_auto_delete_glue_record(): timeout=10 ) assert response.status_code == 200 - + response = requests.delete( URL_BASE + "domains/me/nycu-dev/test-glue-rec", headers = headers, @@ -213,6 +168,8 @@ def test_ttl(ttl, answer): headers = headers, timeout=10 ) + assert response.status_code == 200 + test_ttl(1, 5) test_ttl(10, 10) test_ttl(86401, 5) @@ -222,3 +179,4 @@ def test_ttl(ttl, answer): headers = headers, timeout=10 ) + assert response.status_code == 200 diff --git a/tests/test_ddns.py b/tests/test_ddns.py index da51301..5bf11d2 100644 --- a/tests/test_ddns.py +++ b/tests/test_ddns.py @@ -16,7 +16,7 @@ ("test2-ddns.nycu-dev.me", 'A', "140.113.69.69", 86400), ] -def test_add_A_record(): +def test_add_a_record(): domains = {} for testcase in testdata_A: ddns.add_record(*testcase) diff --git a/tests/test_domain_expired.py b/tests/test_domain_expired.py index 033cae8..2f1af16 100644 --- a/tests/test_domain_expired.py +++ b/tests/test_domain_expired.py @@ -1,17 +1,14 @@ +import logging +import time +from datetime import datetime, timedelta from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from datetime import datetime -from datetime import timedelta -import time -import logging from models import Domains, Records, Users, Glues, db, DDNS from services import DNSService import config - from launch_thread import recycle - ddns = DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") sql_engine = create_engine("sqlite:///:memory:") @@ -27,7 +24,6 @@ dnsService = DNSService(logging, users, domains, glues, records, ddns, config.HOST_DOMAINS) def test_domain_expire(): - # Insert an expiring domain exp_date = datetime.now() + timedelta(seconds=5) domain = db.Domain(userId="109550028", domain="test-expire.nycu-dev.me", @@ -36,7 +32,7 @@ def test_domain_expire(): status=1) session.add(domain) session.commit() - # Waiting for expiring + time.sleep(10) recycle(dnsService) - assert dnsService.get_domain("test-expire.nycu-dev.me") == None + assert dnsService.get_domain("test-expire.nycu-dev.me") is None diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index 76822d4..8fc9cf3 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -1,8 +1,8 @@ +import time +import logging from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -import logging import pydig -import time from models import Domains, Records, Users, Glues, db, DDNS from services import DNSService @@ -89,11 +89,11 @@ def test_glue_record(): dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == {"1.1.1.1"} - + dnsService.del_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() - + # check if glue record is be removed after domain released dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) dnsService.release_domain("test-glue.nycu-dev.me") diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py index d4bdcc1..730bf9f 100644 --- a/tests/test_issue_token.py +++ b/tests/test_issue_token.py @@ -1,6 +1,6 @@ +import logging from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -import logging from services.auth_service import AuthService from models import Users, Domains, db @@ -23,7 +23,7 @@ def test_issue_token(): token = "Bearer " + authService.issue_token(testcase) assert authService.authenticate_token(token) is not None # test modified token - assert authService.authenticate_token(token + 'a') == None + assert authService.authenticate_token(token + 'a') is None # test if data is written session.expire_all()# flush orm cache data = session.query(db.User).filter_by(id=testcase['username']).all() From a7ebc873db3726070c8d2396cf75be5917b6629f Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sun, 3 Dec 2023 23:29:29 +0800 Subject: [PATCH 45/93] Update pr_to_backend.yml Modify: From automatically push to send pull request --- .github/workflows/pr_to_backend.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_to_backend.yml b/.github/workflows/pr_to_backend.yml index d193d5a..7245ec4 100644 --- a/.github/workflows/pr_to_backend.yml +++ b/.github/workflows/pr_to_backend.yml @@ -27,6 +27,13 @@ jobs: git config user.name "GitHub Actions - update submodules" git add --all git commit -m "Update submodules" || echo "No changes to commit" - git push - + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.TOKEN }} + commit-message: Update submodules + title: '[AUTO] Update submodules' + body: 'Automated changes by GitHub Actions. Please review before merging.' + branch: 'update-submodules-${{ github.run_number }}' + base: 'main' From c8412c13388108e54f369587200405761d303523 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sun, 3 Dec 2023 23:30:55 +0800 Subject: [PATCH 46/93] Revert "upgrade pylint score" --- .pylintrc | 84 +++++++++++++++++++++++++++++++-- controllers/auth.py | 7 ++- controllers/ddns.py | 22 ++++----- controllers/domains.py | 18 +++---- controllers/glue.py | 33 +++++-------- launch_thread.py | 44 +++++++++-------- main.py | 32 ++++++++----- models/ddns.py | 13 ++--- models/domains.py | 5 +- models/elastic.py | 13 ++--- models/glues.py | 21 +++------ models/records.py | 12 ++--- models/users.py | 2 +- services/dns_service.py | 29 ++++++------ services/nctu_oauth/__init__.py | 3 +- tests/test_controllers.py | 54 ++++++++++++++++++--- tests/test_ddns.py | 2 +- tests/test_domain_expired.py | 14 ++++-- tests/test_domain_register.py | 8 ++-- tests/test_issue_token.py | 4 +- 20 files changed, 264 insertions(+), 156 deletions(-) diff --git a/.pylintrc b/.pylintrc index 98f0c2b..e766285 100644 --- a/.pylintrc +++ b/.pylintrc @@ -60,12 +60,90 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=duplicate-code, +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape, + duplicate-code, missing-function-docstring, missing-module-docstring, missing-class-docstring, - too-many-arguments, - too-few-public-methods + too-many-arguments # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/controllers/auth.py b/controllers/auth.py index a45bcb7..1ef3b6e 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -1,5 +1,6 @@ from flask import request, g from main import env_test, app, nycu_oauth, authService, dnsService +import config @app.before_request def before_request(): @@ -12,14 +13,16 @@ def get_token(code): token = nycu_oauth.get_token(code) if token: return {'token': authService.issue_token(nycu_oauth.get_profile(token))} - return {'msg': "Invalid code."}, 401 + else: + return {'msg': "Invalid code."}, 401 @app.route("/test_auth/", methods = ['GET']) def get_token_for_test(): if env_test: return {'msg': 'ok', 'token': authService.issue_token(request.json)} - return {'msg': "It is not currently running on testing mode."}, 401 + else: + return {'msg': "It is not currently running on testing mode."}, 401 @app.route("/whoami/", methods = ['GET']) def whoami(): diff --git a/controllers/ddns.py b/controllers/ddns.py index acd044e..b138042 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -22,17 +22,14 @@ def is_domain(domain): def check_type(type_, value): - error_response = None - if type_ not in {'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS'}: - error_response = { + return { "errorType": "DNSError", "msg": f"Not allowed type {type_}." }, 403 - if type_ == 'A': if not is_ip(value, ipaddress.IPv4Address): - error_response = { + return { "errorType": "DNSError", "msg": "Type A with non-IPv4 value." }, 403 @@ -41,7 +38,7 @@ def check_type(type_, value): if type_ == 'AAAA': if not is_ip(value, ipaddress.IPv6Address): - error_response = { + return { "errorType": "DNSError", "msg": "Type AAAA with non-IPv6 value." }, 403 @@ -49,31 +46,28 @@ def check_type(type_, value): value = is_ip(value, ipaddress.IPv6Address) if type_ == 'CNAME' and not is_domain(value): - error_response = { + return { "errorType": "DNSError", "msg": "Type CNAME with non-domain-name value." }, 403 if type_ == 'MX' and not is_domain(value): - error_response = { + return { "errorType": "DNSError", "msg": "Type MX with non-domain-name value." }, 403 if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): - error_response = { + return { "errorType": "DNSError", "msg": "Type TXT with value longer than 255 chars or more than 1 line." }, 403 if type_ == 'NS' and not is_domain(value): - error_response = { - "errorType": "DNSError", - "msg": "Type NS with non-domain-name value." - }, 403 + return {"errorType": "DNSError", "msg": "Type NS with non-domain-name value."}, 403 - return error_response + return None @app.route("/ddns//records//", methods=['POST']) def add_record(domain, type_, value): diff --git a/controllers/domains.py b/controllers/domains.py index 93dfb5f..7cc3bf7 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -1,5 +1,5 @@ -import datetime from flask import g +import datetime from main import app, authService, dnsService, elastic from services import Operation @@ -7,9 +7,9 @@ def list_domains(): if not g.user: return {"msg": "Unauth."}, 401 - if not g.user['isAdmin']: + if not g.user['isAdmin']: return {"msg": "Unauth."}, 403 - + return {"msg": "ok", "data": dnsService.list_domains()} @app.route("/domains/", methods=['POST']) @@ -27,7 +27,7 @@ def register_domain(domain): return {"msg": "Not valid domain name."}, 400 try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.APPLY, domain_name) dnsService.register_domain(g.user['uid'], domain_name) return {"msg": "ok"} @@ -41,9 +41,9 @@ def release_domain(domain): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) - + try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.RELEASE, domain_name) dnsService.release_domain(domain_name) return {"msg": "ok"} @@ -59,7 +59,7 @@ def renew_domain(domain): domain_name = '.'.join(reversed(domain_struct)) try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.RENEW, domain_name) dnsService.renew_domain(domain_name) return {"msg": "ok"} @@ -78,7 +78,7 @@ def get_domain_traffic(domain): today = datetime.date.today() try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) for i in range(29, -1, -1): past_date = today - datetime.timedelta(days=i) @@ -88,3 +88,5 @@ def get_domain_traffic(domain): return {"msg": "ok", "data": result} except Exception as e: return {"msg": str(e)}, 403 + + diff --git a/controllers/glue.py b/controllers/glue.py index 8bb71c6..18d3bfe 100644 --- a/controllers/glue.py +++ b/controllers/glue.py @@ -5,6 +5,7 @@ from main import app, authService, dnsService from services import Operation + domain_regex = re.compile( r'^([A-Za-z0-9]\.|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]\.){1,3}[A-Za-z]{2,6}$' ) @@ -21,17 +22,14 @@ def is_domain(domain): def check_type(type_, value): - error_response = None - if type_ not in {'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS'}: - error_response = { + return { "errorType": "DNSError", "msg": f"Not allowed type {type_}." }, 403 - if type_ == 'A': if not is_ip(value, ipaddress.IPv4Address): - error_response = { + return { "errorType": "DNSError", "msg": "Type A with non-IPv4 value." }, 403 @@ -40,7 +38,7 @@ def check_type(type_, value): if type_ == 'AAAA': if not is_ip(value, ipaddress.IPv6Address): - error_response = { + return { "errorType": "DNSError", "msg": "Type AAAA with non-IPv6 value." }, 403 @@ -48,36 +46,30 @@ def check_type(type_, value): value = is_ip(value, ipaddress.IPv6Address) if type_ == 'CNAME' and not is_domain(value): - error_response = { + return { "errorType": "DNSError", "msg": "Type CNAME with non-domain-name value." }, 403 if type_ == 'MX' and not is_domain(value): - error_response = { + return { "errorType": "DNSError", "msg": "Type MX with non-domain-name value." }, 403 if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): - error_response = { + return { "errorType": "DNSError", "msg": "Type TXT with value longer than 255 chars or more than 1 line." }, 403 if type_ == 'NS' and not is_domain(value): - error_response = { - "errorType": "DNSError", - "msg": "Type NS with non-domain-name value." - }, 403 + return {"errorType": "DNSError", "msg": "Type NS with non-domain-name value."}, 403 - return error_response + return None -@app.route( - "/glue//records///", - methods=['POST'] -) +@app.route("/glue//records///", methods=['POST']) def add_glue_record(domain, subdomain, type_, value): if not g.user: return {"msg": "Unauth."}, 401 @@ -108,10 +100,7 @@ def add_glue_record(domain, subdomain, type_, value): except Exception as e: return {"msg": str(e)}, 403 -@app.route( - "/glue//records///", - methods=['DELETE'] -) +@app.route("/glue//records///", methods=['DELETE']) def del_glue_record(domain, subdomain, type_, value): if not g.user: return {"msg": "Unauth."}, 401 diff --git a/launch_thread.py b/launch_thread.py index a6f1456..f8ecbd9 100644 --- a/launch_thread.py +++ b/launch_thread.py @@ -7,38 +7,42 @@ from models import Users, Domains, Records, Glues, DDNS, db from services import AuthService, DNSService -def recycle(local_dns_service): +def recycle(dnsService): try: - while (domain := local_dns_service.get_expired_domain()) is not None: - logging.info("recycling %s", domain.domain) - local_dns_service.release_domain(domain.domain) + while (domain := dnsService.get_expired_domain()) != None: + logging.info(f"recycling {domain.domain}") + dnsService.release_domain(domain.domain) except Exception: pass -ENV_TEST = os.getenv('TEST') +env_test = os.getenv('TEST') -SQL_ENGINE = None -if ENV_TEST is not None: - SQL_ENGINE = create_engine("sqlite:///:memory:") - db.Base.metadata.create_all(SQL_ENGINE) +sql_engine = None +if env_test is not None: + sql_engine = create_engine("sqlite:///:memory:") + db.Base.metadata.create_all(sql_engine) else: - connection_string = ( - f"mysql+pymysql://{config.MYSQL_USER}:{config.MYSQL_PSWD}" - f"@{config.MYSQL_HOST}/{config.MYSQL_DB}" + sql_engine = create_engine( + 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( + user=config.MYSQL_USER, + pswd=config.MYSQL_PSWD, + host=config.MYSQL_HOST, + db=config.MYSQL_DB + ) ) - SQL_ENGINE = create_engine(connection_string) + ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) -users = Users(SQL_ENGINE) -domains = Domains(SQL_ENGINE) -records = Records(SQL_ENGINE) -glues = Glues(SQL_ENGINE) +users = Users(sql_engine) +domains = Domains(sql_engine) +records = Records(sql_engine) +glues = Glues(sql_engine) -auth_service = AuthService(logging, config.JWT_SECRET, users, domains) -dns_service = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) +authService = AuthService(logging, config.JWT_SECRET, users, domains) +dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) if __name__ == "__main__": while True: - recycle(dns_service) + recycle(dnsService) time.sleep(1) diff --git a/main.py b/main.py index 6c491af..e922460 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ import logging import os +import time from flask import Flask import flask_cors from sqlalchemy import create_engine @@ -13,23 +14,26 @@ app = Flask(__name__) flask_cors.CORS(app) -SQL_ENGINE = None +sql_engine = None if env_test is not None: - SQL_ENGINE = create_engine("sqlite:///:memory:") - db.Base.metadata.create_all(SQL_ENGINE) + sql_engine = create_engine("sqlite:///:memory:") + db.Base.metadata.create_all(sql_engine) else: - connection_string = ( - f"mysql+pymysql://{config.MYSQL_USER}:{config.MYSQL_PSWD}" - f"@{config.MYSQL_HOST}/{config.MYSQL_DB}" + sql_engine = create_engine( + 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( + user=config.MYSQL_USER, + pswd=config.MYSQL_PSWD, + host=config.MYSQL_HOST, + db=config.MYSQL_DB + ) ) - sql_engine = create_engine(connection_string) ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) -users = Users(SQL_ENGINE) -domains = Domains(SQL_ENGINE) -records = Records(SQL_ENGINE) -glues = Glues(SQL_ENGINE) +users = Users(sql_engine) +domains = Domains(sql_engine) +records = Records(sql_engine) +glues = Glues(sql_engine) nycu_oauth = Oauth(redirect_uri = config.NYCU_OAUTH_RURL, client_id = config.NYCU_OAUTH_ID, @@ -39,4 +43,8 @@ authService = AuthService(logging, config.JWT_SECRET, users, domains) dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) -from controllers import auth, domains, ddns, glue # pylint: disable=all +@app.route("/") +def index(): + return "Hello World!" + +from controllers import auth, domains, ddns, glue diff --git a/models/ddns.py b/models/ddns.py index 5bd86cb..e4a164f 100644 --- a/models/ddns.py +++ b/models/ddns.py @@ -6,13 +6,12 @@ class DDNS: - def __launch(self): - pr = subprocess.Popen( # pylint: disable=all + def __launch(self): + pr = subprocess.Popen( ['nsupdate', '-k', self.key_file], bufsize=0, stdin=subprocess.PIPE, - stdout=subprocess.PIPE - ) + stdout=subprocess.PIPE) if self.name_server: pr.stdin.write(f"server {self.name_server}\n".encode()) @@ -58,13 +57,11 @@ def __init__(self, logger, key_file, name_server, zone): def add_record(self, domain, rectype, value, ttl = 5): if domain != "" and rectype != "" and value != "": if rectype == "TXT": - value = value.replace('"', '\"') - value = f'"{value}"' + value = '"%s"' % value.replace('"', '\"') self.queue.put(f"update add {domain} {ttl} {rectype} {value}") def del_record(self, domain, rectype, value): if domain != "": if rectype == "TXT": - value = value.replace('"', '\"') - value = f'"{value}"' + value = '"%s"' % value.replace('"', '\"') self.queue.put(f"update delete {domain} {rectype} {value}") diff --git a/models/domains.py b/models/domains.py index 75c6095..7ec8f5c 100644 --- a/models/domains.py +++ b/models/domains.py @@ -20,10 +20,7 @@ def get_expired_domain(self): session = self.make_session() try: now = datetime.now() - domain = session.query(db.Domain)\ - .filter_by(status=1)\ - .filter(db.Domain.expDate < now)\ - .first() + domain = session.query(db.Domain).filter_by(status=1).filter(db.Domain.expDate < now).first() return domain finally: session.close() diff --git a/models/elastic.py b/models/elastic.py index 3d32b37..fc6c334 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -1,5 +1,6 @@ -import json import requests +import json +import config class Elastic(): @@ -29,13 +30,9 @@ def query(self, domain, date): } } - response = requests.get( - url, - headers=headers, - auth=auth, - data=json.dumps(data), - timeout=3 - ) + response = requests.get(url, headers=headers, auth=auth, data=json.dumps(data)) response_json = response.json() return response_json.get('hits', {}).get('total', {}).get('value', 0) + + diff --git a/models/glues.py b/models/glues.py index 5227460..4961d5a 100644 --- a/models/glues.py +++ b/models/glues.py @@ -6,24 +6,24 @@ class Glues: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.session = scoped_session(sessionmaker(bind=self.sql_engine)) + self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) def get_record(self, glue_id): - session = self.session() + session = self.Session() try: return session.query(db.Glue).filter_by(id=glue_id, status=1).first() finally: session.close() def get_records(self, domain_id): - session = self.session() + session = self.Session() try: return session.query(db.Glue).filter_by(domain=domain_id, status=1).all() finally: session.close() def get_record_by_type_value(self, domain_id, subdomain, type_, value): - session = self.session() + session = self.Session() try: return session.query(db.Glue).filter_by(domain=domain_id, subdomain=subdomain, @@ -34,16 +34,9 @@ def get_record_by_type_value(self, domain_id, subdomain, type_, value): session.close() def add_record(self, domain_id, subdomain, type_, value, ttl): - session = self.session() + session = self.Session() try: - new_record = db.Glue( - domain=domain_id, - subdomain=subdomain, - type=type_, value=value, - ttl=ttl, - status=1, - regDate=datetime.now() - ) + new_record = db.Glue(domain=domain_id, subdomain=subdomain, type=type_, value=value, ttl=ttl, status=1, regDate=datetime.now()) session.add(new_record) session.commit() return new_record @@ -54,7 +47,7 @@ def add_record(self, domain_id, subdomain, type_, value, ttl): session.close() def del_record(self, glue_id): - session = self.session() + session = self.Session() try: record_to_delete = session.query(db.Glue).filter_by(id=glue_id).first() if record_to_delete: diff --git a/models/records.py b/models/records.py index 44f08a2..d45064e 100644 --- a/models/records.py +++ b/models/records.py @@ -6,24 +6,24 @@ class Records: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.session_maker = scoped_session(sessionmaker(bind=self.sql_engine)) + self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) def get_record(self, record_id): - session = self.session_maker() + session = self.Session() try: return session.query(db.Record).filter_by(id=record_id, status=1).first() finally: session.close() def get_records(self, domain_id): - session = self.session_maker() + session = self.Session() try: return session.query(db.Record).filter_by(domain=domain_id, status=1).all() finally: session.close() def get_record_by_type_value(self, domain_id, type_, value): - session = self.session_maker() + session = self.Session() try: return session.query(db.Record).filter_by(domain=domain_id, type=type_, @@ -33,7 +33,7 @@ def get_record_by_type_value(self, domain_id, type_, value): session.close() def add_record(self, domain_id, record_type, value, ttl): - session = self.session_maker() + session = self.Session() try: record = db.Record(domain=domain_id, type=record_type, @@ -50,7 +50,7 @@ def add_record(self, domain_id, record_type, value, ttl): session.close() def del_record_by_id(self, record_id): - session = self.session_maker() + session = self.Session() try: record = session.query(db.Record).filter_by(id=record_id).first() if record: diff --git a/models/users.py b/models/users.py index d642dd2..a94b73c 100644 --- a/models/users.py +++ b/models/users.py @@ -21,7 +21,7 @@ def query(self, uid): def add(self, uid, name, username, password, status, email): session = self.session_factory() try: - user = db.User(id=uid, name=name, username=username, password=password, + user = db.User(id=uid, name=name, username=username, password=password, status=status, email=email) session.add(user) session.commit() diff --git a/services/dns_service.py b/services/dns_service.py index 6c05e85..d1a215c 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -1,15 +1,17 @@ +import time import re from enum import Enum + DOMAIN_REGEX = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(? len(struct): return 0 - for i, element in enumerate(rule): - if element == '*': + for i in range(len(rule)): + if rule[i] == '*': return i + 1 - if element != struct[i]: + elif rule[i] != struct[i]: return 0 - return None - for domain in self.host_domains: - match_result = is_match(domain, domain_struct) - if match_result is not None: - return match_result - - return None + if (x:=is_match(domain, domain_struct)): + return x def get_domain(self, domain_name): domain = self.domains.get_domain(domain_name) @@ -171,10 +168,12 @@ def del_glue_record(self, domain_name, subdomain, type_, value): ) self.glues.del_record(glue_record.id) self.ddns.del_record(real_domain, type_, value) - + def list_domains(self): domains = self.domains.list_all() result = [] for domain in domains: result.append(self.__get_domain_info(domain)) return result + + diff --git a/services/nctu_oauth/__init__.py b/services/nctu_oauth/__init__.py index dc86616..465ec8f 100644 --- a/services/nctu_oauth/__init__.py +++ b/services/nctu_oauth/__init__.py @@ -1,2 +1,3 @@ #-*- encoding: UTF-8 -*- -from .oauth import Oauth + +from .oauth import Oauth \ No newline at end of file diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 4877c11..8fd26c2 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -11,6 +11,51 @@ ], ) +def test_add_ttl(): + + headers = get_headers("109550032") + + def test_ttl(ttl, answer): + # Add record for ttl 10 + response = requests.post( + URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", + json = {'ttl': ttl}, + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + response = requests.get( + URL_BASE + "whoami/", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + for domain in json.loads(response.text)['domains']: + if domain['domain'] == 'test-ttl1.nycu-dev.me': + assert domain['records'][0][3] == answer + response = requests.delete( + URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", + headers = headers, + timeout=10 + ) + + # Register domains + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-ttl1", + headers = headers, + timeout=10 + ) + + test_ttl(1, 5) + test_ttl(10, 10) + test_ttl(86401, 5) + test_ttl("random_string", 5) + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-ttl1", + headers = headers, + timeout=10 + ) + def test_register_and_release_domain(): headers = get_headers("109550004") # Register domains @@ -103,7 +148,7 @@ def test_add_and_delete_records(): assert response.status_code == 200 def test_auto_delete_glue_record(): - + headers = get_headers("110550029") response = requests.post( @@ -112,7 +157,7 @@ def test_auto_delete_glue_record(): timeout=10 ) assert response.status_code == 200 - + response = requests.post( URL_BASE + "glue/me/nycu-dev/test-glue-rec/records/abc/A/1.1.1.1", headers = headers, @@ -126,7 +171,7 @@ def test_auto_delete_glue_record(): timeout=10 ) assert response.status_code == 200 - + response = requests.delete( URL_BASE + "domains/me/nycu-dev/test-glue-rec", headers = headers, @@ -168,8 +213,6 @@ def test_ttl(ttl, answer): headers = headers, timeout=10 ) - assert response.status_code == 200 - test_ttl(1, 5) test_ttl(10, 10) test_ttl(86401, 5) @@ -179,4 +222,3 @@ def test_ttl(ttl, answer): headers = headers, timeout=10 ) - assert response.status_code == 200 diff --git a/tests/test_ddns.py b/tests/test_ddns.py index 5bf11d2..da51301 100644 --- a/tests/test_ddns.py +++ b/tests/test_ddns.py @@ -16,7 +16,7 @@ ("test2-ddns.nycu-dev.me", 'A', "140.113.69.69", 86400), ] -def test_add_a_record(): +def test_add_A_record(): domains = {} for testcase in testdata_A: ddns.add_record(*testcase) diff --git a/tests/test_domain_expired.py b/tests/test_domain_expired.py index 2f1af16..033cae8 100644 --- a/tests/test_domain_expired.py +++ b/tests/test_domain_expired.py @@ -1,14 +1,17 @@ -import logging -import time -from datetime import datetime, timedelta from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from datetime import datetime +from datetime import timedelta +import time +import logging from models import Domains, Records, Users, Glues, db, DDNS from services import DNSService import config + from launch_thread import recycle + ddns = DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") sql_engine = create_engine("sqlite:///:memory:") @@ -24,6 +27,7 @@ dnsService = DNSService(logging, users, domains, glues, records, ddns, config.HOST_DOMAINS) def test_domain_expire(): + # Insert an expiring domain exp_date = datetime.now() + timedelta(seconds=5) domain = db.Domain(userId="109550028", domain="test-expire.nycu-dev.me", @@ -32,7 +36,7 @@ def test_domain_expire(): status=1) session.add(domain) session.commit() - + # Waiting for expiring time.sleep(10) recycle(dnsService) - assert dnsService.get_domain("test-expire.nycu-dev.me") is None + assert dnsService.get_domain("test-expire.nycu-dev.me") == None diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index 8fc9cf3..76822d4 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -1,8 +1,8 @@ -import time -import logging from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +import logging import pydig +import time from models import Domains, Records, Users, Glues, db, DDNS from services import DNSService @@ -89,11 +89,11 @@ def test_glue_record(): dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == {"1.1.1.1"} - + dnsService.del_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() - + # check if glue record is be removed after domain released dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) dnsService.release_domain("test-glue.nycu-dev.me") diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py index 730bf9f..d4bdcc1 100644 --- a/tests/test_issue_token.py +++ b/tests/test_issue_token.py @@ -1,6 +1,6 @@ -import logging from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +import logging from services.auth_service import AuthService from models import Users, Domains, db @@ -23,7 +23,7 @@ def test_issue_token(): token = "Bearer " + authService.issue_token(testcase) assert authService.authenticate_token(token) is not None # test modified token - assert authService.authenticate_token(token + 'a') is None + assert authService.authenticate_token(token + 'a') == None # test if data is written session.expire_all()# flush orm cache data = session.query(db.User).filter_by(id=testcase['username']).all() From ffb07c5efab35c1700b50e70d436022f6c3719d5 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sun, 3 Dec 2023 23:40:40 +0800 Subject: [PATCH 47/93] Update pr_to_backend.yml: add pr description to pr --- .github/workflows/pr_to_backend.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_to_backend.yml b/.github/workflows/pr_to_backend.yml index 7245ec4..f42a353 100644 --- a/.github/workflows/pr_to_backend.yml +++ b/.github/workflows/pr_to_backend.yml @@ -33,7 +33,9 @@ jobs: with: token: ${{ secrets.TOKEN }} commit-message: Update submodules - title: '[AUTO] Update submodules' - body: 'Automated changes by GitHub Actions. Please review before merging.' + title: '[AUTO] Update submodules from ${{ github.ref }}' + body: | + Automated changes by GitHub Actions. Please review before merging. + Triggered by: ${{ github.ref }} branch: 'update-submodules-${{ github.run_number }}' base: 'main' From 65321a43e5bef6f7549812dbf2dd78ba8e9f2fdb Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sun, 3 Dec 2023 23:43:19 +0800 Subject: [PATCH 48/93] Update pr_to_backend.yml: add commit message to pr --- .github/workflows/pr_to_backend.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr_to_backend.yml b/.github/workflows/pr_to_backend.yml index f42a353..760edfa 100644 --- a/.github/workflows/pr_to_backend.yml +++ b/.github/workflows/pr_to_backend.yml @@ -33,9 +33,8 @@ jobs: with: token: ${{ secrets.TOKEN }} commit-message: Update submodules - title: '[AUTO] Update submodules from ${{ github.ref }}' + title: '[AUTO] Update submodules' body: | Automated changes by GitHub Actions. Please review before merging. - Triggered by: ${{ github.ref }} branch: 'update-submodules-${{ github.run_number }}' base: 'main' From 8652abb862cbb41188cdfac95ba37ed4150315e9 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 4 Dec 2023 00:33:01 +0800 Subject: [PATCH 49/93] Create deploy.yml --- .github/workflows/deploy.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..21ae273 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,16 @@ +name: Trigger Jenkins Webhook + +on: + push: + branches: + - main + +jobs: + trigger-webhook: + runs-on: ubuntu-latest + + steps: + - name: Trigger Jenkins Webhook + uses: wei/curl@v1 + with: + args: -X POST http://103.179.29.10:9983/job/backend-deploy/build?token=1103d87793920e9cd3826b418bd2efbd9c From 1622e20425f33b240a6575ceac5a67d0c6cd2d9b Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 4 Dec 2023 00:35:32 +0800 Subject: [PATCH 50/93] Delete .github/workflows/deploy.yml --- .github/workflows/deploy.yml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 21ae273..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Trigger Jenkins Webhook - -on: - push: - branches: - - main - -jobs: - trigger-webhook: - runs-on: ubuntu-latest - - steps: - - name: Trigger Jenkins Webhook - uses: wei/curl@v1 - with: - args: -X POST http://103.179.29.10:9983/job/backend-deploy/build?token=1103d87793920e9cd3826b418bd2efbd9c From 96bf91d0935a029ef4235b5d0265c21d4b7e2446 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 4 Dec 2023 00:58:45 +0800 Subject: [PATCH 51/93] Update pr_to_backend.yml --- .github/workflows/pr_to_backend.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr_to_backend.yml b/.github/workflows/pr_to_backend.yml index 760edfa..36ca1c5 100644 --- a/.github/workflows/pr_to_backend.yml +++ b/.github/workflows/pr_to_backend.yml @@ -4,6 +4,9 @@ on: push: branches: - main + pull_request: + branches: + - main jobs: update: From 0e867e15a09742c7c2dcc88eaef775295ebee8b7 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Mon, 4 Dec 2023 09:33:53 -0500 Subject: [PATCH 52/93] feat: add flask logger --- controllers/auth.py | 6 ++++++ main.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/controllers/auth.py b/controllers/auth.py index 1ef3b6e..71681de 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -7,6 +7,12 @@ def before_request(): g.user = authService.authenticate_token(request.headers.get('Authorization')) + extra = { + 'remote_addr': request.remote_addr, + 'url': request.url + } + app.logger.info('Logged in', extra=extra) + @app.route("/oauth/", methods = ['GET']) def get_token(code): diff --git a/main.py b/main.py index e922460..d30cf2d 100644 --- a/main.py +++ b/main.py @@ -13,6 +13,13 @@ app = Flask(__name__) flask_cors.CORS(app) +app.logger.setLevel(logging.INFO) +handler = logging.StreamHandler() +formatter = logging.Formatter( + '%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d] [%(remote_addr)s] [%(url)s]' +) +handler.setFormatter(formatter) +app.logger.addHandler(handler) sql_engine = None if env_test is not None: From 080ffa8fa7e16a46a91405a777d33cbb584244cc Mon Sep 17 00:00:00 2001 From: roger Date: Tue, 5 Dec 2023 10:35:38 +0800 Subject: [PATCH 53/93] improve pylint score --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 6c491af..22d9cc1 100644 --- a/main.py +++ b/main.py @@ -22,7 +22,7 @@ f"mysql+pymysql://{config.MYSQL_USER}:{config.MYSQL_PSWD}" f"@{config.MYSQL_HOST}/{config.MYSQL_DB}" ) - sql_engine = create_engine(connection_string) + SQL_ENGINE = create_engine(connection_string) ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) From 75306147afacbec0b9b866e5a98f43f42204a727 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 5 Dec 2023 10:39:17 +0800 Subject: [PATCH 54/93] Revert "Revert "upgrade pylint score" due to some db connection bug" --- .pylintrc | 84 ++------------------------------- controllers/auth.py | 7 +-- controllers/ddns.py | 22 +++++---- controllers/domains.py | 18 ++++--- controllers/glue.py | 33 ++++++++----- launch_thread.py | 44 ++++++++--------- main.py | 32 +++++-------- models/ddns.py | 13 +++-- models/domains.py | 5 +- models/elastic.py | 13 +++-- models/glues.py | 21 ++++++--- models/records.py | 12 ++--- models/users.py | 2 +- services/dns_service.py | 29 ++++++------ services/nctu_oauth/__init__.py | 3 +- tests/test_controllers.py | 54 +++------------------ tests/test_ddns.py | 2 +- tests/test_domain_expired.py | 14 ++---- tests/test_domain_register.py | 8 ++-- tests/test_issue_token.py | 4 +- 20 files changed, 156 insertions(+), 264 deletions(-) diff --git a/.pylintrc b/.pylintrc index e766285..98f0c2b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -60,90 +60,12 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, - duplicate-code, +disable=duplicate-code, missing-function-docstring, missing-module-docstring, missing-class-docstring, - too-many-arguments + too-many-arguments, + too-few-public-methods # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/controllers/auth.py b/controllers/auth.py index 71681de..e2a5560 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -1,6 +1,5 @@ from flask import request, g from main import env_test, app, nycu_oauth, authService, dnsService -import config @app.before_request def before_request(): @@ -19,16 +18,14 @@ def get_token(code): token = nycu_oauth.get_token(code) if token: return {'token': authService.issue_token(nycu_oauth.get_profile(token))} - else: - return {'msg': "Invalid code."}, 401 + return {'msg': "Invalid code."}, 401 @app.route("/test_auth/", methods = ['GET']) def get_token_for_test(): if env_test: return {'msg': 'ok', 'token': authService.issue_token(request.json)} - else: - return {'msg': "It is not currently running on testing mode."}, 401 + return {'msg': "It is not currently running on testing mode."}, 401 @app.route("/whoami/", methods = ['GET']) def whoami(): diff --git a/controllers/ddns.py b/controllers/ddns.py index b138042..acd044e 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -22,14 +22,17 @@ def is_domain(domain): def check_type(type_, value): + error_response = None + if type_ not in {'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS'}: - return { + error_response = { "errorType": "DNSError", "msg": f"Not allowed type {type_}." }, 403 + if type_ == 'A': if not is_ip(value, ipaddress.IPv4Address): - return { + error_response = { "errorType": "DNSError", "msg": "Type A with non-IPv4 value." }, 403 @@ -38,7 +41,7 @@ def check_type(type_, value): if type_ == 'AAAA': if not is_ip(value, ipaddress.IPv6Address): - return { + error_response = { "errorType": "DNSError", "msg": "Type AAAA with non-IPv6 value." }, 403 @@ -46,28 +49,31 @@ def check_type(type_, value): value = is_ip(value, ipaddress.IPv6Address) if type_ == 'CNAME' and not is_domain(value): - return { + error_response = { "errorType": "DNSError", "msg": "Type CNAME with non-domain-name value." }, 403 if type_ == 'MX' and not is_domain(value): - return { + error_response = { "errorType": "DNSError", "msg": "Type MX with non-domain-name value." }, 403 if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): - return { + error_response = { "errorType": "DNSError", "msg": "Type TXT with value longer than 255 chars or more than 1 line." }, 403 if type_ == 'NS' and not is_domain(value): - return {"errorType": "DNSError", "msg": "Type NS with non-domain-name value."}, 403 + error_response = { + "errorType": "DNSError", + "msg": "Type NS with non-domain-name value." + }, 403 - return None + return error_response @app.route("/ddns//records//", methods=['POST']) def add_record(domain, type_, value): diff --git a/controllers/domains.py b/controllers/domains.py index 7cc3bf7..93dfb5f 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -1,5 +1,5 @@ -from flask import g import datetime +from flask import g from main import app, authService, dnsService, elastic from services import Operation @@ -7,9 +7,9 @@ def list_domains(): if not g.user: return {"msg": "Unauth."}, 401 - if not g.user['isAdmin']: + if not g.user['isAdmin']: return {"msg": "Unauth."}, 403 - + return {"msg": "ok", "data": dnsService.list_domains()} @app.route("/domains/", methods=['POST']) @@ -27,7 +27,7 @@ def register_domain(domain): return {"msg": "Not valid domain name."}, 400 try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.APPLY, domain_name) dnsService.register_domain(g.user['uid'], domain_name) return {"msg": "ok"} @@ -41,9 +41,9 @@ def release_domain(domain): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) - + try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.RELEASE, domain_name) dnsService.release_domain(domain_name) return {"msg": "ok"} @@ -59,7 +59,7 @@ def renew_domain(domain): domain_name = '.'.join(reversed(domain_struct)) try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.RENEW, domain_name) dnsService.renew_domain(domain_name) return {"msg": "ok"} @@ -78,7 +78,7 @@ def get_domain_traffic(domain): today = datetime.date.today() try: - if not g.user['isAdmin']: + if not g.user['isAdmin']: authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) for i in range(29, -1, -1): past_date = today - datetime.timedelta(days=i) @@ -88,5 +88,3 @@ def get_domain_traffic(domain): return {"msg": "ok", "data": result} except Exception as e: return {"msg": str(e)}, 403 - - diff --git a/controllers/glue.py b/controllers/glue.py index 18d3bfe..8bb71c6 100644 --- a/controllers/glue.py +++ b/controllers/glue.py @@ -5,7 +5,6 @@ from main import app, authService, dnsService from services import Operation - domain_regex = re.compile( r'^([A-Za-z0-9]\.|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]\.){1,3}[A-Za-z]{2,6}$' ) @@ -22,14 +21,17 @@ def is_domain(domain): def check_type(type_, value): + error_response = None + if type_ not in {'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS'}: - return { + error_response = { "errorType": "DNSError", "msg": f"Not allowed type {type_}." }, 403 + if type_ == 'A': if not is_ip(value, ipaddress.IPv4Address): - return { + error_response = { "errorType": "DNSError", "msg": "Type A with non-IPv4 value." }, 403 @@ -38,7 +40,7 @@ def check_type(type_, value): if type_ == 'AAAA': if not is_ip(value, ipaddress.IPv6Address): - return { + error_response = { "errorType": "DNSError", "msg": "Type AAAA with non-IPv6 value." }, 403 @@ -46,30 +48,36 @@ def check_type(type_, value): value = is_ip(value, ipaddress.IPv6Address) if type_ == 'CNAME' and not is_domain(value): - return { + error_response = { "errorType": "DNSError", "msg": "Type CNAME with non-domain-name value." }, 403 if type_ == 'MX' and not is_domain(value): - return { + error_response = { "errorType": "DNSError", "msg": "Type MX with non-domain-name value." }, 403 if type_ == 'TXT' and (len(value) > 255 or value.count('\n')): - return { + error_response = { "errorType": "DNSError", "msg": "Type TXT with value longer than 255 chars or more than 1 line." }, 403 if type_ == 'NS' and not is_domain(value): - return {"errorType": "DNSError", "msg": "Type NS with non-domain-name value."}, 403 + error_response = { + "errorType": "DNSError", + "msg": "Type NS with non-domain-name value." + }, 403 - return None + return error_response -@app.route("/glue//records///", methods=['POST']) +@app.route( + "/glue//records///", + methods=['POST'] +) def add_glue_record(domain, subdomain, type_, value): if not g.user: return {"msg": "Unauth."}, 401 @@ -100,7 +108,10 @@ def add_glue_record(domain, subdomain, type_, value): except Exception as e: return {"msg": str(e)}, 403 -@app.route("/glue//records///", methods=['DELETE']) +@app.route( + "/glue//records///", + methods=['DELETE'] +) def del_glue_record(domain, subdomain, type_, value): if not g.user: return {"msg": "Unauth."}, 401 diff --git a/launch_thread.py b/launch_thread.py index f8ecbd9..a6f1456 100644 --- a/launch_thread.py +++ b/launch_thread.py @@ -7,42 +7,38 @@ from models import Users, Domains, Records, Glues, DDNS, db from services import AuthService, DNSService -def recycle(dnsService): +def recycle(local_dns_service): try: - while (domain := dnsService.get_expired_domain()) != None: - logging.info(f"recycling {domain.domain}") - dnsService.release_domain(domain.domain) + while (domain := local_dns_service.get_expired_domain()) is not None: + logging.info("recycling %s", domain.domain) + local_dns_service.release_domain(domain.domain) except Exception: pass -env_test = os.getenv('TEST') +ENV_TEST = os.getenv('TEST') -sql_engine = None -if env_test is not None: - sql_engine = create_engine("sqlite:///:memory:") - db.Base.metadata.create_all(sql_engine) +SQL_ENGINE = None +if ENV_TEST is not None: + SQL_ENGINE = create_engine("sqlite:///:memory:") + db.Base.metadata.create_all(SQL_ENGINE) else: - sql_engine = create_engine( - 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( - user=config.MYSQL_USER, - pswd=config.MYSQL_PSWD, - host=config.MYSQL_HOST, - db=config.MYSQL_DB - ) + connection_string = ( + f"mysql+pymysql://{config.MYSQL_USER}:{config.MYSQL_PSWD}" + f"@{config.MYSQL_HOST}/{config.MYSQL_DB}" ) - + SQL_ENGINE = create_engine(connection_string) ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) -users = Users(sql_engine) -domains = Domains(sql_engine) -records = Records(sql_engine) -glues = Glues(sql_engine) +users = Users(SQL_ENGINE) +domains = Domains(SQL_ENGINE) +records = Records(SQL_ENGINE) +glues = Glues(SQL_ENGINE) -authService = AuthService(logging, config.JWT_SECRET, users, domains) -dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) +auth_service = AuthService(logging, config.JWT_SECRET, users, domains) +dns_service = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) if __name__ == "__main__": while True: - recycle(dnsService) + recycle(dns_service) time.sleep(1) diff --git a/main.py b/main.py index d30cf2d..afe1fe4 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,5 @@ import logging import os -import time from flask import Flask import flask_cors from sqlalchemy import create_engine @@ -21,26 +20,23 @@ handler.setFormatter(formatter) app.logger.addHandler(handler) -sql_engine = None +SQL_ENGINE = None if env_test is not None: - sql_engine = create_engine("sqlite:///:memory:") - db.Base.metadata.create_all(sql_engine) + SQL_ENGINE = create_engine("sqlite:///:memory:") + db.Base.metadata.create_all(SQL_ENGINE) else: - sql_engine = create_engine( - 'mysql+pymysql://{user}:{pswd}@{host}/{db}'.format( - user=config.MYSQL_USER, - pswd=config.MYSQL_PSWD, - host=config.MYSQL_HOST, - db=config.MYSQL_DB - ) + connection_string = ( + f"mysql+pymysql://{config.MYSQL_USER}:{config.MYSQL_PSWD}" + f"@{config.MYSQL_HOST}/{config.MYSQL_DB}" ) + sql_engine = create_engine(connection_string) ddns = DDNS(logging, config.DDNS_KEY, config.DDNS_SERVER, config.DDNS_ZONE) -users = Users(sql_engine) -domains = Domains(sql_engine) -records = Records(sql_engine) -glues = Glues(sql_engine) +users = Users(SQL_ENGINE) +domains = Domains(SQL_ENGINE) +records = Records(SQL_ENGINE) +glues = Glues(SQL_ENGINE) nycu_oauth = Oauth(redirect_uri = config.NYCU_OAUTH_RURL, client_id = config.NYCU_OAUTH_ID, @@ -50,8 +46,4 @@ authService = AuthService(logging, config.JWT_SECRET, users, domains) dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) -@app.route("/") -def index(): - return "Hello World!" - -from controllers import auth, domains, ddns, glue +from controllers import auth, domains, ddns, glue # pylint: disable=all diff --git a/models/ddns.py b/models/ddns.py index e4a164f..5bd86cb 100644 --- a/models/ddns.py +++ b/models/ddns.py @@ -6,12 +6,13 @@ class DDNS: - def __launch(self): - pr = subprocess.Popen( + def __launch(self): + pr = subprocess.Popen( # pylint: disable=all ['nsupdate', '-k', self.key_file], bufsize=0, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + stdout=subprocess.PIPE + ) if self.name_server: pr.stdin.write(f"server {self.name_server}\n".encode()) @@ -57,11 +58,13 @@ def __init__(self, logger, key_file, name_server, zone): def add_record(self, domain, rectype, value, ttl = 5): if domain != "" and rectype != "" and value != "": if rectype == "TXT": - value = '"%s"' % value.replace('"', '\"') + value = value.replace('"', '\"') + value = f'"{value}"' self.queue.put(f"update add {domain} {ttl} {rectype} {value}") def del_record(self, domain, rectype, value): if domain != "": if rectype == "TXT": - value = '"%s"' % value.replace('"', '\"') + value = value.replace('"', '\"') + value = f'"{value}"' self.queue.put(f"update delete {domain} {rectype} {value}") diff --git a/models/domains.py b/models/domains.py index 7ec8f5c..75c6095 100644 --- a/models/domains.py +++ b/models/domains.py @@ -20,7 +20,10 @@ def get_expired_domain(self): session = self.make_session() try: now = datetime.now() - domain = session.query(db.Domain).filter_by(status=1).filter(db.Domain.expDate < now).first() + domain = session.query(db.Domain)\ + .filter_by(status=1)\ + .filter(db.Domain.expDate < now)\ + .first() return domain finally: session.close() diff --git a/models/elastic.py b/models/elastic.py index fc6c334..3d32b37 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -1,6 +1,5 @@ -import requests import json -import config +import requests class Elastic(): @@ -30,9 +29,13 @@ def query(self, domain, date): } } - response = requests.get(url, headers=headers, auth=auth, data=json.dumps(data)) + response = requests.get( + url, + headers=headers, + auth=auth, + data=json.dumps(data), + timeout=3 + ) response_json = response.json() return response_json.get('hits', {}).get('total', {}).get('value', 0) - - diff --git a/models/glues.py b/models/glues.py index 4961d5a..5227460 100644 --- a/models/glues.py +++ b/models/glues.py @@ -6,24 +6,24 @@ class Glues: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) + self.session = scoped_session(sessionmaker(bind=self.sql_engine)) def get_record(self, glue_id): - session = self.Session() + session = self.session() try: return session.query(db.Glue).filter_by(id=glue_id, status=1).first() finally: session.close() def get_records(self, domain_id): - session = self.Session() + session = self.session() try: return session.query(db.Glue).filter_by(domain=domain_id, status=1).all() finally: session.close() def get_record_by_type_value(self, domain_id, subdomain, type_, value): - session = self.Session() + session = self.session() try: return session.query(db.Glue).filter_by(domain=domain_id, subdomain=subdomain, @@ -34,9 +34,16 @@ def get_record_by_type_value(self, domain_id, subdomain, type_, value): session.close() def add_record(self, domain_id, subdomain, type_, value, ttl): - session = self.Session() + session = self.session() try: - new_record = db.Glue(domain=domain_id, subdomain=subdomain, type=type_, value=value, ttl=ttl, status=1, regDate=datetime.now()) + new_record = db.Glue( + domain=domain_id, + subdomain=subdomain, + type=type_, value=value, + ttl=ttl, + status=1, + regDate=datetime.now() + ) session.add(new_record) session.commit() return new_record @@ -47,7 +54,7 @@ def add_record(self, domain_id, subdomain, type_, value, ttl): session.close() def del_record(self, glue_id): - session = self.Session() + session = self.session() try: record_to_delete = session.query(db.Glue).filter_by(id=glue_id).first() if record_to_delete: diff --git a/models/records.py b/models/records.py index d45064e..44f08a2 100644 --- a/models/records.py +++ b/models/records.py @@ -6,24 +6,24 @@ class Records: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.Session = scoped_session(sessionmaker(bind=self.sql_engine)) + self.session_maker = scoped_session(sessionmaker(bind=self.sql_engine)) def get_record(self, record_id): - session = self.Session() + session = self.session_maker() try: return session.query(db.Record).filter_by(id=record_id, status=1).first() finally: session.close() def get_records(self, domain_id): - session = self.Session() + session = self.session_maker() try: return session.query(db.Record).filter_by(domain=domain_id, status=1).all() finally: session.close() def get_record_by_type_value(self, domain_id, type_, value): - session = self.Session() + session = self.session_maker() try: return session.query(db.Record).filter_by(domain=domain_id, type=type_, @@ -33,7 +33,7 @@ def get_record_by_type_value(self, domain_id, type_, value): session.close() def add_record(self, domain_id, record_type, value, ttl): - session = self.Session() + session = self.session_maker() try: record = db.Record(domain=domain_id, type=record_type, @@ -50,7 +50,7 @@ def add_record(self, domain_id, record_type, value, ttl): session.close() def del_record_by_id(self, record_id): - session = self.Session() + session = self.session_maker() try: record = session.query(db.Record).filter_by(id=record_id).first() if record: diff --git a/models/users.py b/models/users.py index a94b73c..d642dd2 100644 --- a/models/users.py +++ b/models/users.py @@ -21,7 +21,7 @@ def query(self, uid): def add(self, uid, name, username, password, status, email): session = self.session_factory() try: - user = db.User(id=uid, name=name, username=username, password=password, + user = db.User(id=uid, name=name, username=username, password=password, status=status, email=email) session.add(user) session.commit() diff --git a/services/dns_service.py b/services/dns_service.py index d1a215c..6c05e85 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -1,17 +1,15 @@ -import time import re from enum import Enum - DOMAIN_REGEX = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(? len(struct): return 0 - for i in range(len(rule)): - if rule[i] == '*': + for i, element in enumerate(rule): + if element == '*': return i + 1 - elif rule[i] != struct[i]: + if element != struct[i]: return 0 + return None + for domain in self.host_domains: - if (x:=is_match(domain, domain_struct)): - return x + match_result = is_match(domain, domain_struct) + if match_result is not None: + return match_result + + return None def get_domain(self, domain_name): domain = self.domains.get_domain(domain_name) @@ -168,12 +171,10 @@ def del_glue_record(self, domain_name, subdomain, type_, value): ) self.glues.del_record(glue_record.id) self.ddns.del_record(real_domain, type_, value) - + def list_domains(self): domains = self.domains.list_all() result = [] for domain in domains: result.append(self.__get_domain_info(domain)) return result - - diff --git a/services/nctu_oauth/__init__.py b/services/nctu_oauth/__init__.py index 465ec8f..dc86616 100644 --- a/services/nctu_oauth/__init__.py +++ b/services/nctu_oauth/__init__.py @@ -1,3 +1,2 @@ #-*- encoding: UTF-8 -*- - -from .oauth import Oauth \ No newline at end of file +from .oauth import Oauth diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 8fd26c2..4877c11 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -11,51 +11,6 @@ ], ) -def test_add_ttl(): - - headers = get_headers("109550032") - - def test_ttl(ttl, answer): - # Add record for ttl 10 - response = requests.post( - URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", - json = {'ttl': ttl}, - headers = headers, - timeout=10 - ) - assert response.status_code == 200 - response = requests.get( - URL_BASE + "whoami/", - headers = headers, - timeout=10 - ) - assert response.status_code == 200 - for domain in json.loads(response.text)['domains']: - if domain['domain'] == 'test-ttl1.nycu-dev.me': - assert domain['records'][0][3] == answer - response = requests.delete( - URL_BASE + "ddns/me/nycu-dev/test-ttl1/records/A/140.113.89.64", - headers = headers, - timeout=10 - ) - - # Register domains - response = requests.post( - URL_BASE + "domains/me/nycu-dev/test-ttl1", - headers = headers, - timeout=10 - ) - - test_ttl(1, 5) - test_ttl(10, 10) - test_ttl(86401, 5) - test_ttl("random_string", 5) - response = requests.delete( - URL_BASE + "domains/me/nycu-dev/test-ttl1", - headers = headers, - timeout=10 - ) - def test_register_and_release_domain(): headers = get_headers("109550004") # Register domains @@ -148,7 +103,7 @@ def test_add_and_delete_records(): assert response.status_code == 200 def test_auto_delete_glue_record(): - + headers = get_headers("110550029") response = requests.post( @@ -157,7 +112,7 @@ def test_auto_delete_glue_record(): timeout=10 ) assert response.status_code == 200 - + response = requests.post( URL_BASE + "glue/me/nycu-dev/test-glue-rec/records/abc/A/1.1.1.1", headers = headers, @@ -171,7 +126,7 @@ def test_auto_delete_glue_record(): timeout=10 ) assert response.status_code == 200 - + response = requests.delete( URL_BASE + "domains/me/nycu-dev/test-glue-rec", headers = headers, @@ -213,6 +168,8 @@ def test_ttl(ttl, answer): headers = headers, timeout=10 ) + assert response.status_code == 200 + test_ttl(1, 5) test_ttl(10, 10) test_ttl(86401, 5) @@ -222,3 +179,4 @@ def test_ttl(ttl, answer): headers = headers, timeout=10 ) + assert response.status_code == 200 diff --git a/tests/test_ddns.py b/tests/test_ddns.py index da51301..5bf11d2 100644 --- a/tests/test_ddns.py +++ b/tests/test_ddns.py @@ -16,7 +16,7 @@ ("test2-ddns.nycu-dev.me", 'A', "140.113.69.69", 86400), ] -def test_add_A_record(): +def test_add_a_record(): domains = {} for testcase in testdata_A: ddns.add_record(*testcase) diff --git a/tests/test_domain_expired.py b/tests/test_domain_expired.py index 033cae8..2f1af16 100644 --- a/tests/test_domain_expired.py +++ b/tests/test_domain_expired.py @@ -1,17 +1,14 @@ +import logging +import time +from datetime import datetime, timedelta from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from datetime import datetime -from datetime import timedelta -import time -import logging from models import Domains, Records, Users, Glues, db, DDNS from services import DNSService import config - from launch_thread import recycle - ddns = DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") sql_engine = create_engine("sqlite:///:memory:") @@ -27,7 +24,6 @@ dnsService = DNSService(logging, users, domains, glues, records, ddns, config.HOST_DOMAINS) def test_domain_expire(): - # Insert an expiring domain exp_date = datetime.now() + timedelta(seconds=5) domain = db.Domain(userId="109550028", domain="test-expire.nycu-dev.me", @@ -36,7 +32,7 @@ def test_domain_expire(): status=1) session.add(domain) session.commit() - # Waiting for expiring + time.sleep(10) recycle(dnsService) - assert dnsService.get_domain("test-expire.nycu-dev.me") == None + assert dnsService.get_domain("test-expire.nycu-dev.me") is None diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index 76822d4..8fc9cf3 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -1,8 +1,8 @@ +import time +import logging from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -import logging import pydig -import time from models import Domains, Records, Users, Glues, db, DDNS from services import DNSService @@ -89,11 +89,11 @@ def test_glue_record(): dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == {"1.1.1.1"} - + dnsService.del_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") time.sleep(5) assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() - + # check if glue record is be removed after domain released dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) dnsService.release_domain("test-glue.nycu-dev.me") diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py index d4bdcc1..730bf9f 100644 --- a/tests/test_issue_token.py +++ b/tests/test_issue_token.py @@ -1,6 +1,6 @@ +import logging from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -import logging from services.auth_service import AuthService from models import Users, Domains, db @@ -23,7 +23,7 @@ def test_issue_token(): token = "Bearer " + authService.issue_token(testcase) assert authService.authenticate_token(token) is not None # test modified token - assert authService.authenticate_token(token + 'a') == None + assert authService.authenticate_token(token + 'a') is None # test if data is written session.expire_all()# flush orm cache data = session.query(db.User).filter_by(id=testcase['username']).all() From 402054552be3b9ebe0da9437e63821c4b3811aee Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 6 Dec 2023 01:15:13 -0500 Subject: [PATCH 55/93] using a better way to query elasticsearch --- models/elastic.py | 50 ++++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/models/elastic.py b/models/elastic.py index 3d32b37..e67cf78 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -1,41 +1,33 @@ -import json -import requests - +from elasticsearch import Elasticsearch +import logging class Elastic(): def __init__(self, server, user, password): - self.server = server - self.user = user - self.password = password + self.elastic = Elasticsearch( + server, + http_auth=(user, password) + ) def query(self, domain, date): - url = f"http://{self.server}:9200/fluentd.named.dns/_search?pretty=true" - headers = {"Content-Type": "application/json"} - auth = (self.user, self.password) - - data = { - "size": 0, + query = { "query": { "bool": { - "must": [ - {"match_phrase": {"log": f"({domain})"}}, + "should": [ + {"wildcard": {"log": f"(*.{domain})"}}, + {"wildcard": {"log": f"({domain})"}} ], - "filter": [ - { - "term": {"log_time": date} + "filter": { + "range": { + "@timestamp": { + "gte": f"{date}T00:00:00", + "lt": f"{date}T23:59:59" + } } - ] + } } } - } - - response = requests.get( - url, - headers=headers, - auth=auth, - data=json.dumps(data), - timeout=3 - ) - response_json = response.json() + + count_response = self.elastic.count(index="fluentd.named.dns", body=query) + return count_response['count'] + - return response_json.get('hits', {}).get('total', {}).get('value', 0) From 153ca6530c13a8a2cbce2d6ecf6c6ca2704ecc1a Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Wed, 6 Dec 2023 14:19:37 +0800 Subject: [PATCH 56/93] Update pr_to_backend.yml --- .github/workflows/pr_to_backend.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pr_to_backend.yml b/.github/workflows/pr_to_backend.yml index 36ca1c5..760edfa 100644 --- a/.github/workflows/pr_to_backend.yml +++ b/.github/workflows/pr_to_backend.yml @@ -4,9 +4,6 @@ on: push: branches: - main - pull_request: - branches: - - main jobs: update: From 3303650e0ec18cb1bbde50d1c9fa94075ddbcea1 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Wed, 6 Dec 2023 14:19:37 +0800 Subject: [PATCH 57/93] Update pr_to_backend.yml --- models/elastic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/elastic.py b/models/elastic.py index e67cf78..1c0f172 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -26,7 +26,7 @@ def query(self, domain, date): } } } - + } count_response = self.elastic.count(index="fluentd.named.dns", body=query) return count_response['count'] From c9ec9c0f5f0f5ca968516e223adbde4e58a712ed Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 6 Dec 2023 03:47:30 -0500 Subject: [PATCH 58/93] resolve conflict --- models/elastic.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/models/elastic.py b/models/elastic.py index 1c0f172..407e62f 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -1,5 +1,5 @@ from elasticsearch import Elasticsearch -import logging + class Elastic(): def __init__(self, server, user, password): @@ -27,7 +27,5 @@ def query(self, domain, date): } } } - count_response = self.elastic.count(index="fluentd.named.dns", body=query) + count_response = self.elastic.count(body=query, index="fluentd.named.dns") # pylint: disable=unexpected-keyword-arg return count_response['count'] - - From 284d93c8f0d71cf0b29c578df7d82771f3099984 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 6 Dec 2023 08:59:22 -0500 Subject: [PATCH 59/93] debug: elastic always returns sum of queries --- models/elastic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/models/elastic.py b/models/elastic.py index 407e62f..b167423 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -12,9 +12,10 @@ def query(self, domain, date): query = { "query": { "bool": { - "should": [ - {"wildcard": {"log": f"(*.{domain})"}}, - {"wildcard": {"log": f"({domain})"}} + "must": [ + {"match": + {"log": f"{domain}"} + } ], "filter": { "range": { From 9f25b924694469daad379d2e32103736df039b10 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Sun, 10 Dec 2023 09:16:01 -0500 Subject: [PATCH 60/93] fix bug: cannot add mx record --- models/ddns.py | 5 +++++ tests/test_ddns.py | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/models/ddns.py b/models/ddns.py index 5bd86cb..89f8d39 100644 --- a/models/ddns.py +++ b/models/ddns.py @@ -60,6 +60,8 @@ def add_record(self, domain, rectype, value, ttl = 5): if rectype == "TXT": value = value.replace('"', '\"') value = f'"{value}"' + if rectype == "MX": + value = f"10 {value}" self.queue.put(f"update add {domain} {ttl} {rectype} {value}") def del_record(self, domain, rectype, value): @@ -67,4 +69,7 @@ def del_record(self, domain, rectype, value): if rectype == "TXT": value = value.replace('"', '\"') value = f'"{value}"' + if rectype == "MX": + value = f"10 {value}" self.queue.put(f"update delete {domain} {rectype} {value}") + print(f"update delete {domain} {rectype} {value}") diff --git a/tests/test_ddns.py b/tests/test_ddns.py index 5bf11d2..c030174 100644 --- a/tests/test_ddns.py +++ b/tests/test_ddns.py @@ -15,6 +15,7 @@ ("test-ddns.nycu-dev.me", 'A', "140.113.64.89", 5), ("test2-ddns.nycu-dev.me", 'A', "140.113.69.69", 86400), ] +testdata_mx = ("test-ddns.nycu-dev.me", 'MX', "test-ddns.nycu-dev.me", 5) def test_add_a_record(): domains = {} @@ -31,3 +32,11 @@ def test_add_a_record(): time.sleep(5) for domain in domains: assert not resolver.query(domain, 'A') + +def test_add_mx_record(): + ddns.add_record(*testdata_mx) + time.sleep(5) + assert set(resolver.query(testdata_mx[0], 'MX')) == {"10 test-ddns.nycu-dev.me."} + ddns.del_record(*testdata_mx[:-1]) + time.sleep(5) + assert set(resolver.query(testdata_mx[0], 'MX')) == set() From 07534e54ce814281b09b318aca394628ff064151 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Sun, 10 Dec 2023 10:36:33 -0500 Subject: [PATCH 61/93] Add API: query domain by id. --- controllers/domains.py | 11 +++++++++++ services/dns_service.py | 8 ++++++++ tests/test_controllers.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/controllers/domains.py b/controllers/domains.py index 93dfb5f..97c2fb6 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -66,6 +66,17 @@ def renew_domain(domain): except Exception as e: return {"msg": str(e)}, 403 +@app.route("/domain/", methods=['GET']) +def get_domain_by_id(domain_id): + + if not domain_id.isnumeric(): + return {"msg": "Invalid id."}, 400 + + domain = dnsService.get_domain_by_id(int(domain_id)) + if domain is None: + return {"msg": "No such entry."}, 404 + return {"msg": "ok", "domain": domain['domain']} + @app.route("/traffic/", methods=['GET']) def get_domain_traffic(domain): if not g.user: diff --git a/services/dns_service.py b/services/dns_service.py index 6c05e85..be941ca 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -87,6 +87,14 @@ def get_domain(self, domain_name): domain_info = self.__get_domain_info(domain) return domain_info + def get_domain_by_id(self, idx): + domain = self.domains.get_domain_by_id(idx) + if not domain: + return None + + domain_info = self.__get_domain_info(domain) + return domain_info + def get_expired_domain(self): return self.domains.get_expired_domain() diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 4877c11..3330001 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -62,6 +62,34 @@ def test_register_and_release_domain(): domains = json.loads(response.text)['domains'] assert {domain['domain'] for domain in domains} == set() +def test_get_domain_by_id(): + headers = get_headers("0716023") + # Register domain + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-domain-id", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + + # Because of the parallel, we cannot determine + # which one would be the answer. + response = requests.get( + URL_BASE + "domain/1", + headers = headers, + timeout=10 + ) + domain_name = json.loads(response.text)['domain'] + assert response.status_code == 200 + assert domain_name != "" + + # Release domain + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-domain-id", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 def test_add_and_delete_records(): headers = get_headers("109550028") # Register domains From 9eb12be662737e45fb25148d4d8470c67fd71f94 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Sun, 10 Dec 2023 21:39:39 -0500 Subject: [PATCH 62/93] base64 encoding in txt record. --- controllers/ddns.py | 7 +++++++ controllers/glue.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/controllers/ddns.py b/controllers/ddns.py index acd044e..92f3396 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -1,3 +1,4 @@ +import base64 import re import ipaddress from flask import request, g @@ -83,6 +84,9 @@ def add_record(domain, type_, value): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) + if type_ == 'TXT': + value = base64.b64decode(value).decode() + try: req = request.json if req and 'ttl' in req and 5 <= int(req['ttl']) <= 86400: @@ -111,6 +115,9 @@ def del_record(domain, type_, value): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) + if type_ == 'TXT': + value = base64.b64decode(value).decode() + check_result = check_type(type_, value) if check_result: return check_result diff --git a/controllers/glue.py b/controllers/glue.py index 8bb71c6..6a203bf 100644 --- a/controllers/glue.py +++ b/controllers/glue.py @@ -1,3 +1,4 @@ +import base64 import re import ipaddress from flask import request, g @@ -85,6 +86,9 @@ def add_glue_record(domain, subdomain, type_, value): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) + if type_ == 'TXT': + value = base64.b64decode(value).decode() + if dnsService.check_domain(f"{domain_name}.{subdomain}"): return {"msg": "Not valid subdomain."}, 400 @@ -119,6 +123,9 @@ def del_glue_record(domain, subdomain, type_, value): domain_struct = domain.lower().strip('/').split('/') domain_name = '.'.join(reversed(domain_struct)) + if type_ == 'TXT': + value = base64.b64decode(value).decode() + check_result = check_type(type_, value) if check_result: return check_result From 6e7e63f8f4cfd0b029e2eb06c5283c8ef3ac9210 Mon Sep 17 00:00:00 2001 From: roger Date: Mon, 11 Dec 2023 21:37:31 +0800 Subject: [PATCH 63/93] feat. login via email --- controllers/auth.py | 38 +++++++++++++++++++++++++++++++++----- main.py | 5 ++++- models/db.py | 2 +- models/users.py | 13 +++++++++++++ services/__init__.py | 1 + services/auth_service.py | 17 ++++++++++++++++- services/mail_service.py | 28 ++++++++++++++++++++++++++++ tests/test_issue_token.py | 2 +- tests/test_login.py | 32 ++++++++++++++++++++++++++++++++ 9 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 services/mail_service.py create mode 100644 tests/test_login.py diff --git a/controllers/auth.py b/controllers/auth.py index e2a5560..7d12c69 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -1,30 +1,32 @@ from flask import request, g -from main import env_test, app, nycu_oauth, authService, dnsService +from main import env_test, app, nycu_oauth, authService, dnsService, mailService, BASE_URL @app.before_request def before_request(): g.user = authService.authenticate_token(request.headers.get('Authorization')) + if g.user and g.user['type'] != "logged": + g.user = None extra = { 'remote_addr': request.remote_addr, 'url': request.url } - app.logger.info('Logged in', extra=extra) + app.logger.info("Logged in", extra=extra) @app.route("/oauth/", methods = ['GET']) def get_token(code): token = nycu_oauth.get_token(code) if token: - return {'token': authService.issue_token(nycu_oauth.get_profile(token))} + return {'token': authService.issue_token(nycu_oauth.get_profile(token), "logged")} return {'msg': "Invalid code."}, 401 @app.route("/test_auth/", methods = ['GET']) def get_token_for_test(): if env_test: - return {'msg': 'ok', 'token': authService.issue_token(request.json)} + return {'msg': "ok", 'token': authService.issue_token(request.json, "logged")} return {'msg': "It is not currently running on testing mode."}, 401 @app.route("/whoami/", methods = ['GET']) @@ -35,4 +37,30 @@ def whoami(): data['email'] = g.user['email'] data['domains'] = dnsService.list_domains_by_user(g.user['uid']) return data - return {"msg": "Unauth."}, 401 + return {'msg': "Unauth."}, 401 + +@app.route("/login_email", methods=['POST']) +def login_email(): + try: + data = request.json + if 'email' not in data: + return {"msg": "Invalid data."}, 400 + except Exception: + return {"msg": "Invalid data."}, 400 + + if not authService.verify_email(data['email']): + return {"msg": "Invalid email."}, 400 + + # passwd = ''.join(random.sample(string.ascii_letters + string.digits, 8)) + # hashed_passwd = bcrypt.hashpw(passwd.encode(), bcrypt.gensalt()) + # users.update_passwd(data['email'], hashed_passwd.decode()) + token = authService.issue_token( + { + 'email' : data['email'], + 'username': data['email'] + }, + "logged" + ) + + mailService.send_mail(data['email'], "NYCU-ME 登入鏈結", f"{BASE_URL}login_email/?token={token}") + return {'msg': 'ok'} diff --git a/main.py b/main.py index ef19d4c..8d5d3c9 100644 --- a/main.py +++ b/main.py @@ -6,7 +6,7 @@ import config from models import Users, Domains, Records, Glues, DDNS, Elastic, db -from services import AuthService, DNSService, Oauth +from services import AuthService, DNSService, MailService, Oauth env_test = os.getenv('TEST') @@ -20,6 +20,8 @@ handler.setFormatter(formatter) app.logger.addHandler(handler) +BASE_URL = config.BASE_URL + SQL_ENGINE = None if env_test is not None: SQL_ENGINE = create_engine("sqlite:///:memory:") @@ -45,5 +47,6 @@ elastic = Elastic(config.ELASTICSERVER, config.ELASTICUSER, config.ELASTICPASS) authService = AuthService(logging, config.JWT_SECRET, users, domains) dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) +mailService = MailService(logging, config.SMTP_SERVER, config.SMTP_PORT, config.SMTP_USER, config.SMTP_PASS, config.SMTP_FROM) from controllers import auth, domains, ddns, glue # pylint: disable=all diff --git a/models/db.py b/models/db.py index 18952db..1981b40 100644 --- a/models/db.py +++ b/models/db.py @@ -7,7 +7,7 @@ class User(Base): __tablename__ = 'users' - id = Column(String(16), primary_key=True) + id = Column(String(256), primary_key=True) name = Column(String(256), nullable=False) username = Column(String(256), nullable=False) password = Column(String(100), nullable=False, default='') diff --git a/models/users.py b/models/users.py index d642dd2..3722057 100644 --- a/models/users.py +++ b/models/users.py @@ -43,3 +43,16 @@ def update_email(self, uid, email): session.rollback() finally: session.close() + + def update_password(self, uid, password): + session = self.session_factory() + try: + user = session.query(db.User).filter_by(id=uid).first() + if user: + user.password = password + session.commit() + except Exception as e: + logging.error("Error updating user password: %s", e) + session.rollback() + finally: + session.close() diff --git a/services/__init__.py b/services/__init__.py index c474b5d..511b057 100644 --- a/services/__init__.py +++ b/services/__init__.py @@ -1,3 +1,4 @@ from .auth_service import * from .dns_service import * from .nctu_oauth import * +from .mail_service import * diff --git a/services/auth_service.py b/services/auth_service.py index b0c4c54..3babe45 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -1,5 +1,7 @@ from datetime import timezone, datetime from enum import Enum +from email_validator import validate_email, EmailNotValidError +import bcrypt import jwt class Operation(Enum): @@ -25,7 +27,7 @@ def __init__(self, logger, jwt_secret, users, domains): self.users = users self.domains = domains - def issue_token(self, profile): + def issue_token(self, profile, type_): now = int(datetime.now(tz=timezone.utc).timestamp()) token = profile token['iss'] = 'dns.nycu.me' @@ -33,6 +35,7 @@ def issue_token(self, profile): token['iat'] = token['nbf'] = now token['uid'] = token['username'] token['isAdmin'] = False + token['type'] = type_ user = self.users.query(token['uid']) @@ -91,3 +94,15 @@ def authorize_action(self, uid, action, domain_name): raise UnauthorizedError( f"You cannot modify domain {domain_name} which you don't have." ) + + def verify_email(self, email): + try: + if not validate_email(email): + return False + if email.endswith('nycu.edu.tw') or email.endswith('nctu.edu.tw'): + return False + if email.endswith('.edu.tw'): + return True + return False + except EmailNotValidError: + return False diff --git a/services/mail_service.py b/services/mail_service.py new file mode 100644 index 0000000..6ca75d5 --- /dev/null +++ b/services/mail_service.py @@ -0,0 +1,28 @@ +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +class MailService: + def __init__(self, logger, smtp_server, smtp_port, smtp_user, smtp_pass, smtp_from): + self.logger = logger + self.smtp_server = smtp_server + self.smtp_port = smtp_port + self.smtp_user = smtp_user + self.smtp_pass = smtp_pass + self.smtp_from = smtp_from + + def send_mail(self, to, subject, body): + try: + message = MIMEMultipart() + message["From"] = self.smtp_from + message["To"] = to + message["Subject"] = subject + message.attach(MIMEText(body, "plain")) + + with smtplib.SMTP(self.smtp_server, self.smtp_port) as server: + server.starttls() + server.login(self.smtp_user, self.smtp_pass) + server.sendmail(self.smtp_from, to, message.as_string()) + self.logger.debug("Email sent successfully") + except Exception as e: + self.logger.warning("Error sending email: %s", str(e)) diff --git a/tests/test_issue_token.py b/tests/test_issue_token.py index 730bf9f..5aa860a 100644 --- a/tests/test_issue_token.py +++ b/tests/test_issue_token.py @@ -20,7 +20,7 @@ def test_issue_token(): for testcase in testdata: - token = "Bearer " + authService.issue_token(testcase) + token = "Bearer " + authService.issue_token(testcase, "logged") assert authService.authenticate_token(token) is not None # test modified token assert authService.authenticate_token(token + 'a') is None diff --git a/tests/test_login.py b/tests/test_login.py new file mode 100644 index 0000000..c9065bd --- /dev/null +++ b/tests/test_login.py @@ -0,0 +1,32 @@ +import requests +from .test_common import URL_BASE + +def test_email_login(): + response = requests.post( + URL_BASE + "login_email", + json = {'email': "10360874@me.mcu.edu.tw"}, + timeout=10 + ) + print(response.text) + assert response.status_code == 200 + + response = requests.post( + URL_BASE + "login_email", + json = {'email': "rogerdeng92@gmail.com"}, + timeout=10 + ) + assert response.status_code == 400 + + response = requests.post( + URL_BASE + "login_email", + json = {'email': "lin.cs09@nycu.edu.tw"}, + timeout=10 + ) + assert response.status_code == 400 + + response = requests.post( + URL_BASE + "login_email", + json = {'email': "ccy@cs.nctu.edu.tw"}, + timeout=10 + ) + assert response.status_code == 400 From e0d96eeefa58fe6f0eb2827a75875f6383184a08 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Mon, 11 Dec 2023 21:53:01 +0800 Subject: [PATCH 64/93] Update auth_service.py --- services/auth_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/auth_service.py b/services/auth_service.py index 3babe45..beccd9a 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -1,7 +1,6 @@ from datetime import timezone, datetime from enum import Enum from email_validator import validate_email, EmailNotValidError -import bcrypt import jwt class Operation(Enum): From dbb8f567b3047325f1cd5fbe6db3f3809e78710e Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Mon, 11 Dec 2023 09:08:45 -0500 Subject: [PATCH 65/93] Fix bug: sometimes get_domain_by_id would fail. --- tests/test_controllers.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 3330001..660ce3a 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -72,16 +72,23 @@ def test_get_domain_by_id(): ) assert response.status_code == 200 + response = requests.get( + URL_BASE + "whoami/", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + idx = json.loads(response.text)['domains'][0]['id'] # Because of the parallel, we cannot determine # which one would be the answer. response = requests.get( - URL_BASE + "domain/1", + URL_BASE + f"domain/{idx}", headers = headers, timeout=10 ) - domain_name = json.loads(response.text)['domain'] assert response.status_code == 200 - assert domain_name != "" + domain_name = json.loads(response.text)['domain'] + assert domain_name == "test-domain-id.nycu-dev.me" # Release domain response = requests.delete( From 1827c177df5bf116222f5e60e7ead4c4a6ff6073 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 12 Dec 2023 22:00:02 -0500 Subject: [PATCH 66/93] modify db schema --- models/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/db.py b/models/db.py index 1981b40..cfb8d63 100644 --- a/models/db.py +++ b/models/db.py @@ -23,7 +23,7 @@ class Domain(Base): __tablename__ = 'domains' id = Column(Integer, primary_key=True, autoincrement=True) - userId = Column(String(16), ForeignKey('users.id'), nullable=False) + userId = Column(String(256), ForeignKey('users.id'), nullable=False) domain = Column(Text) regDate = Column(DateTime) expDate = Column(DateTime) From 03eef23c7bc7c698e3a387e2fee07e5c458bcf89 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 12 Dec 2023 23:04:14 -0500 Subject: [PATCH 67/93] add minimum elastic search matching score. --- models/elastic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/elastic.py b/models/elastic.py index b167423..e52dca5 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -24,9 +24,10 @@ def query(self, domain, date): "lt": f"{date}T23:59:59" } } - } + }, } - } + }, + "min_score": 7 } count_response = self.elastic.count(body=query, index="fluentd.named.dns") # pylint: disable=unexpected-keyword-arg return count_response['count'] From a348f8fa98e5b278714566e432576cf0e6d3f001 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 13 Dec 2023 04:31:41 +0000 Subject: [PATCH 68/93] modify threholds --- models/elastic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/elastic.py b/models/elastic.py index e52dca5..ee1e6b1 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -27,7 +27,7 @@ def query(self, domain, date): }, } }, - "min_score": 7 + "min_score": 3 } count_response = self.elastic.count(body=query, index="fluentd.named.dns") # pylint: disable=unexpected-keyword-arg return count_response['count'] From a8cf9ae532267f1249af9e8f9ccf8f9220eab133 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Mon, 18 Dec 2023 15:11:15 -0500 Subject: [PATCH 69/93] more human-readable expire time returns --- services/dns_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/dns_service.py b/services/dns_service.py index be941ca..4afad73 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -1,4 +1,5 @@ import re +from datetime import datetime from enum import Enum DOMAIN_REGEX = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(? Date: Mon, 18 Dec 2023 15:35:19 -0500 Subject: [PATCH 70/93] prevent attacker from dumping all the domains via /domain/$idx route. --- controllers/domains.py | 7 ++++++- services/dns_service.py | 1 - tests/test_controllers.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/controllers/domains.py b/controllers/domains.py index 97c2fb6..70d8543 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -75,7 +75,12 @@ def get_domain_by_id(domain_id): domain = dnsService.get_domain_by_id(int(domain_id)) if domain is None: return {"msg": "No such entry."}, 404 - return {"msg": "ok", "domain": domain['domain']} + try: + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain['domain']) + return {"msg": "ok", "domain": domain} + except Exception as e: + return {"msg": str(e)}, 403 @app.route("/traffic/", methods=['GET']) def get_domain_traffic(domain): diff --git a/services/dns_service.py b/services/dns_service.py index 4afad73..e91104a 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -1,5 +1,4 @@ import re -from datetime import datetime from enum import Enum DOMAIN_REGEX = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(? Date: Mon, 18 Dec 2023 20:14:18 -0500 Subject: [PATCH 71/93] administrator can now add/remove records --- controllers/ddns.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/controllers/ddns.py b/controllers/ddns.py index 92f3396..ed2ec30 100644 --- a/controllers/ddns.py +++ b/controllers/ddns.py @@ -101,7 +101,8 @@ def add_record(domain, type_, value): return check_result try: - authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) dnsService.add_record(domain_name, type_, value, ttl) return {"msg": "ok"} except Exception as e: @@ -123,7 +124,8 @@ def del_record(domain, type_, value): return check_result try: - authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) dnsService.del_record(domain_name, type_, value) return {"msg": "ok"} except Exception as e: From dc6c53f8615656931834d7c38a24713d30d711ef Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 19 Dec 2023 00:20:05 -0500 Subject: [PATCH 72/93] debug: user cannot add invalid record such as some special symbols now. --- controllers/glue.py | 2 +- services/dns_service.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controllers/glue.py b/controllers/glue.py index 6a203bf..410ee64 100644 --- a/controllers/glue.py +++ b/controllers/glue.py @@ -89,7 +89,7 @@ def add_glue_record(domain, subdomain, type_, value): if type_ == 'TXT': value = base64.b64decode(value).decode() - if dnsService.check_domain(f"{domain_name}.{subdomain}"): + if dnsService.check_domain(f"{subdomain}.{domain_name}") < len(domain_struct): return {"msg": "Not valid subdomain."}, 400 try: diff --git a/services/dns_service.py b/services/dns_service.py index e91104a..e4abecb 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -70,14 +70,14 @@ def is_match(rule, struct): if element != struct[i]: return 0 - return None + return 0 for domain in self.host_domains: match_result = is_match(domain, domain_struct) - if match_result is not None: + if match_result: return match_result - return None + return 0 def get_domain(self, domain_name): domain = self.domains.get_domain(domain_name) From 9baf9b2b3ab97747b036c36d703be4484ae38d47 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 19 Dec 2023 00:28:19 -0500 Subject: [PATCH 73/93] feat. allow admin to operate glue records --- controllers/glue.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/controllers/glue.py b/controllers/glue.py index 410ee64..e149926 100644 --- a/controllers/glue.py +++ b/controllers/glue.py @@ -106,7 +106,8 @@ def add_glue_record(domain, subdomain, type_, value): return check_result try: - authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) dnsService.add_glue_record(domain_name, subdomain, type_, value, ttl) return {"msg": "ok"} except Exception as e: @@ -131,7 +132,8 @@ def del_glue_record(domain, subdomain, type_, value): return check_result try: - authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) dnsService.del_glue_record(domain_name, subdomain, type_, value) return {"msg": "ok"} except Exception as e: From 9b127dd5ebdc883cf77a85860f9773ad93ba8f76 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 19 Dec 2023 23:07:13 +0800 Subject: [PATCH 74/93] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2c91df0..c1701c8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # backend-flask-server +運行環境在 [backend](https://github.com/NYCU-ME/backend)，此 repo 乃 API server 之原始碼。 + ## Architecture ``` From 7b89dd25ef7952912bb2daff8996ea8e1252ba4f Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 19 Dec 2023 23:10:39 +0800 Subject: [PATCH 75/93] Update README.md --- README.md | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/README.md b/README.md index c1701c8..e8be865 100644 --- a/README.md +++ b/README.md @@ -4,47 +4,6 @@ ## Architecture -``` -. -├── config.py -├── config.py.sample -├── controllers -│   ├── auth.py -│   ├── ddns.py -│   ├── domains.py -│   ├── glue.py -│   └── __init__.py -├── launch_thread.py -├── main.py -├── models -│   ├── db.py -│   ├── ddns.py -│   ├── domains.py -│   ├── elastic.py -│   ├── glues.py -│   ├── __init__.py -│   ├── records.py -│   └── users.py -├── README.md -├── services -│   ├── auth_service.py -│   ├── dns_service.py -│   ├── __init__.py -│   └── nctu_oauth -│   ├── __init__.py -│   ├── oauth.py -│   └── README.md -└── tests - ├── __init__.py - ├── test_attacker.py - ├── test_common.py - ├── test_controllers.py - ├── test_ddns.py - ├── test_domain_expired.py - ├── test_domain_register.py - └── test_issue_token.py -``` - ### config.py 和 config.py.sample: config.py 包含應用程序的配置信息，如資料庫連接設置、API金鑰等。 config.py.sample 是 config.py 的範本，用來展示配置文件的格式和示例值。 From 3d6726e7524403aecd28334d8c3b76fdcfc9acb2 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 19 Dec 2023 23:12:07 +0800 Subject: [PATCH 76/93] Update README.md --- README.md | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/README.md b/README.md index e8be865..79b4cfe 100644 --- a/README.md +++ b/README.md @@ -4,24 +4,4 @@ ## Architecture -### config.py 和 config.py.sample: -config.py 包含應用程序的配置信息，如資料庫連接設置、API金鑰等。 -config.py.sample 是 config.py 的範本，用來展示配置文件的格式和示例值。 - -### controllers: -這個目錄包含了應用程序的控制器或路由層，用於處理HTTP請求和路由它們到適當的功能模組。 - -### launch_thread.py: -這個文件包含一個用於啟動回收 domain 的 process。 - -### main.py: -這個文件是應用程序的入口點，它可能包含主要的程式邏輯，如應用程序的初始化和啟動。 - -### models: -這個目錄包含應用程序的模型層，用於定義資料模型、資料庫表結構以及與資料庫的互動。 - -### services: -這個目錄包含應用程序的服務層，用於實現不同的業務邏輯，如身份驗證服務、DNS服務等。 - -### tests: -這個目錄包含測試用例，用於測試應用程序的各個部分，確保它們按照預期工作。 +MVC 架構加上 Service Layer 與 ORM。 From 008086598ddcd43e621fd03164111dad3a05ce74 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 19 Dec 2023 23:25:35 +0800 Subject: [PATCH 77/93] Update README.md --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 79b4cfe..ee61593 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,14 @@ ## Architecture -MVC 架構加上 Service Layer 與 ORM。 +MVC 架構加上 Service Layer 與 SQLAlchemy ORM。 + +`main.py` 是主程式，`launch_thread` 是 backend_app（詳看 [backend](https://github.com/NYCU-ME/backend)）會開啓的 process，主要工作回收過期 domains。 +`models` 負責與資料庫、資料來源互動，`models/db.sql` 是 ORM 的本體，而 `models/users.py`、`models/domains.py`、`models/records.py`……則類似 Repository Pattern 中的 Repository。 +`services` 中負責簡化 Controller 的邏輯，將一部分邏輯和功能放入 Services，如與 DNS 相關的認證、註冊功能放入 DNSService 中。 +`controllers` 負責處理 API 請求，會使用 services 中寫好的邏輯，在大多數的情況下，`controllers` 不該直接與 `models` 互動。 +`tests` 則存放 Unit Tests。 + +## References + +[Flask Docs](https://flask.palletsprojects.com/en/3.0.x/) From b18a1bf86d742ab1e14d570ff9b7bf0bf78bb0b8 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 19 Dec 2023 23:25:52 +0800 Subject: [PATCH 78/93] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index ee61593..dfe04c8 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,13 @@ MVC 架構加上 Service Layer 與 SQLAlchemy ORM。 `main.py` 是主程式，`launch_thread` 是 backend_app（詳看 [backend](https://github.com/NYCU-ME/backend)）會開啓的 process，主要工作回收過期 domains。 + `models` 負責與資料庫、資料來源互動，`models/db.sql` 是 ORM 的本體，而 `models/users.py`、`models/domains.py`、`models/records.py`……則類似 Repository Pattern 中的 Repository。 + `services` 中負責簡化 Controller 的邏輯，將一部分邏輯和功能放入 Services，如與 DNS 相關的認證、註冊功能放入 DNSService 中。 + `controllers` 負責處理 API 請求，會使用 services 中寫好的邏輯，在大多數的情況下，`controllers` 不該直接與 `models` 互動。 + `tests` 則存放 Unit Tests。 ## References From 3a8846ffa64134521d2c2f6ab722c01597efeb2b Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Tue, 19 Dec 2023 23:15:07 -0500 Subject: [PATCH 79/93] debug: ad authentication for querying domain by id. --- controllers/domains.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/controllers/domains.py b/controllers/domains.py index 70d8543..0e5d6f3 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -72,6 +72,9 @@ def get_domain_by_id(domain_id): if not domain_id.isnumeric(): return {"msg": "Invalid id."}, 400 + if not g.user: + return {"msg": "Unauth."}, 401 + domain = dnsService.get_domain_by_id(int(domain_id)) if domain is None: return {"msg": "No such entry."}, 404 From 5ceef416a0daa776698c347fee95a4bcbe56c9fb Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Tue, 16 Jan 2024 23:13:11 +0800 Subject: [PATCH 80/93] Update elastic.py --- models/elastic.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/models/elastic.py b/models/elastic.py index ee1e6b1..3d4616f 100644 --- a/models/elastic.py +++ b/models/elastic.py @@ -11,23 +11,26 @@ def __init__(self, server, user, password): def query(self, domain, date): query = { "query": { - "bool": { - "must": [ - {"match": - {"log": f"{domain}"} - } - ], + "constant_score": { "filter": { - "range": { - "@timestamp": { - "gte": f"{date}T00:00:00", - "lt": f"{date}T23:59:59" + "bool": { + "must": { + "match": { + "log": f"{domain}" + } + }, + "filter": { + "range": { + "@timestamp": { + "gte": f"{date}T00:00:00", + "lt": f"{date}T23:59:59" + } + } } } - }, + } } - }, - "min_score": 3 + } } count_response = self.elastic.count(body=query, index="fluentd.named.dns") # pylint: disable=unexpected-keyword-arg return count_response['count'] From 091801b321a312c3ade6fd572b894578126ca6fb Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 17 Jan 2024 19:38:45 +0800 Subject: [PATCH 81/93] Disable email login --- controllers/auth.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/controllers/auth.py b/controllers/auth.py index 7d12c69..e214aa1 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -39,28 +39,28 @@ def whoami(): return data return {'msg': "Unauth."}, 401 -@app.route("/login_email", methods=['POST']) -def login_email(): - try: - data = request.json - if 'email' not in data: - return {"msg": "Invalid data."}, 400 - except Exception: - return {"msg": "Invalid data."}, 400 +# @app.route("/login_email", methods=['POST']) +# def login_email(): +# try: +# data = request.json +# if 'email' not in data: +# return {"msg": "Invalid data."}, 400 +# except Exception: +# return {"msg": "Invalid data."}, 400 - if not authService.verify_email(data['email']): - return {"msg": "Invalid email."}, 400 +# if not authService.verify_email(data['email']): +# return {"msg": "Invalid email."}, 400 - # passwd = ''.join(random.sample(string.ascii_letters + string.digits, 8)) - # hashed_passwd = bcrypt.hashpw(passwd.encode(), bcrypt.gensalt()) - # users.update_passwd(data['email'], hashed_passwd.decode()) - token = authService.issue_token( - { - 'email' : data['email'], - 'username': data['email'] - }, - "logged" - ) +# # passwd = ''.join(random.sample(string.ascii_letters + string.digits, 8)) +# # hashed_passwd = bcrypt.hashpw(passwd.encode(), bcrypt.gensalt()) +# # users.update_passwd(data['email'], hashed_passwd.decode()) +# token = authService.issue_token( +# { +# 'email' : data['email'], +# 'username': data['email'] +# }, +# "logged" +# ) - mailService.send_mail(data['email'], "NYCU-ME 登入鏈結", f"{BASE_URL}login_email/?token={token}") - return {'msg': 'ok'} +# mailService.send_mail(data['email'], "NYCU-ME 登入鏈結", f"{BASE_URL}login_email/?token={token}") +# return {'msg': 'ok'} From 962e8901c3b7af3641dd3dc78595cae0277f6a45 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 17 Jan 2024 19:44:08 +0800 Subject: [PATCH 82/93] remove email login test --- tests/test_login.py | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 tests/test_login.py diff --git a/tests/test_login.py b/tests/test_login.py deleted file mode 100644 index c9065bd..0000000 --- a/tests/test_login.py +++ /dev/null @@ -1,32 +0,0 @@ -import requests -from .test_common import URL_BASE - -def test_email_login(): - response = requests.post( - URL_BASE + "login_email", - json = {'email': "10360874@me.mcu.edu.tw"}, - timeout=10 - ) - print(response.text) - assert response.status_code == 200 - - response = requests.post( - URL_BASE + "login_email", - json = {'email': "rogerdeng92@gmail.com"}, - timeout=10 - ) - assert response.status_code == 400 - - response = requests.post( - URL_BASE + "login_email", - json = {'email': "lin.cs09@nycu.edu.tw"}, - timeout=10 - ) - assert response.status_code == 400 - - response = requests.post( - URL_BASE + "login_email", - json = {'email': "ccy@cs.nctu.edu.tw"}, - timeout=10 - ) - assert response.status_code == 400 From fde26827108a27fb3e13edd65082054281a6bab0 Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 17 Jan 2024 06:59:12 -0500 Subject: [PATCH 83/93] fix lint error --- controllers/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/auth.py b/controllers/auth.py index e214aa1..3ba799d 100644 --- a/controllers/auth.py +++ b/controllers/auth.py @@ -1,5 +1,5 @@ from flask import request, g -from main import env_test, app, nycu_oauth, authService, dnsService, mailService, BASE_URL +from main import env_test, app, nycu_oauth, authService, dnsService #, mailService, BASE_URL @app.before_request def before_request(): From 23ef0373826058e032d140be6db843d7eb776172 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Fri, 19 Jan 2024 14:11:55 +0800 Subject: [PATCH 84/93] Update main.py --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 8d5d3c9..f259bd1 100644 --- a/main.py +++ b/main.py @@ -29,7 +29,7 @@ else: connection_string = ( f"mysql+pymysql://{config.MYSQL_USER}:{config.MYSQL_PSWD}" - f"@{config.MYSQL_HOST}/{config.MYSQL_DB}" + f"@{config.MYSQL_HOST}/{config.MYSQL_DB}?wait_timeout=31536000" ) SQL_ENGINE = create_engine(connection_string) From 8e8628fdfdcd42de9d14652895780b06e2c69c33 Mon Sep 17 00:00:00 2001 From: roger Date: Sun, 21 Jan 2024 09:16:15 +0800 Subject: [PATCH 85/93] add simple metrics --- controllers/__init__.py | 1 + controllers/domains.py | 26 +------------------------- controllers/metrics.py | 36 ++++++++++++++++++++++++++++++++++++ main.py | 2 +- models/domains.py | 29 ++++++++++++++++++++--------- models/glues.py | 12 ++++++------ models/records.py | 12 ++++++------ models/users.py | 11 +++++++++++ services/auth_service.py | 3 +++ services/dns_service.py | 3 +++ tests/test_metrics.py | 17 +++++++++++++++++ 11 files changed, 105 insertions(+), 47 deletions(-) create mode 100644 controllers/metrics.py create mode 100644 tests/test_metrics.py diff --git a/controllers/__init__.py b/controllers/__init__.py index 3330676..9f6d6f8 100644 --- a/controllers/__init__.py +++ b/controllers/__init__.py @@ -2,3 +2,4 @@ from .domains import * from .ddns import * from .glue import * +from .metrics import * diff --git a/controllers/domains.py b/controllers/domains.py index 0e5d6f3..d77c4e6 100644 --- a/controllers/domains.py +++ b/controllers/domains.py @@ -1,6 +1,5 @@ -import datetime from flask import g -from main import app, authService, dnsService, elastic +from main import app, authService, dnsService from services import Operation @app.route("/domains", methods=['GET']) @@ -84,26 +83,3 @@ def get_domain_by_id(domain_id): return {"msg": "ok", "domain": domain} except Exception as e: return {"msg": str(e)}, 403 - -@app.route("/traffic/", methods=['GET']) -def get_domain_traffic(domain): - if not g.user: - return {"msg": "Unauth."}, 401 - - domain_struct = domain.lower().strip('/').split('/') - domain_name = '.'.join(reversed(domain_struct)) - - result = [] - today = datetime.date.today() - - try: - if not g.user['isAdmin']: - authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) - for i in range(29, -1, -1): - past_date = today - datetime.timedelta(days=i) - date = past_date.strftime("%Y-%m-%d") - result.append((date, elastic.query(domain_name, date))) - - return {"msg": "ok", "data": result} - except Exception as e: - return {"msg": str(e)}, 403 diff --git a/controllers/metrics.py b/controllers/metrics.py new file mode 100644 index 0000000..ca4c580 --- /dev/null +++ b/controllers/metrics.py @@ -0,0 +1,36 @@ +import datetime +from flask import g +from main import app, authService, dnsService, elastic +from services import Operation + +@app.route("/metrics/", methods=['GET']) +def get_metrics(): + num_of_user = authService.count_user() + num_of_domain = dnsService.count_domain() + return { + 'num_of_user': num_of_user, + 'num_of_domain': num_of_domain + } + +@app.route("/traffic/", methods=['GET']) +def get_domain_traffic(domain): + if not g.user: + return {"msg": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + result = [] + today = datetime.date.today() + + try: + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + for i in range(29, -1, -1): + past_date = today - datetime.timedelta(days=i) + date = past_date.strftime("%Y-%m-%d") + result.append((date, elastic.query(domain_name, date))) + + return {"msg": "ok", "data": result} + except Exception as e: + return {"msg": str(e)}, 403 diff --git a/main.py b/main.py index f259bd1..13ba0c8 100644 --- a/main.py +++ b/main.py @@ -49,4 +49,4 @@ dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) mailService = MailService(logging, config.SMTP_SERVER, config.SMTP_PORT, config.SMTP_USER, config.SMTP_PASS, config.SMTP_FROM) -from controllers import auth, domains, ddns, glue # pylint: disable=all +from controllers import auth, domains, ddns, glue, metrics # pylint: disable=all diff --git a/models/domains.py b/models/domains.py index 75c6095..4ffc8e3 100644 --- a/models/domains.py +++ b/models/domains.py @@ -6,10 +6,10 @@ class Domains: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.make_session = scoped_session(sessionmaker(bind=self.sql_engine)) + self.session_factory = scoped_session(sessionmaker(bind=self.sql_engine)) def get_domain(self, domain_name): - session = self.make_session() + session = self.session_factory() try: domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() return domain @@ -17,7 +17,7 @@ def get_domain(self, domain_name): session.close() def get_expired_domain(self): - session = self.make_session() + session = self.session_factory() try: now = datetime.now() domain = session.query(db.Domain)\ @@ -29,7 +29,7 @@ def get_expired_domain(self): session.close() def get_domain_by_id(self, domain_id): - session = self.make_session() + session = self.session_factory() try: domain = session.query(db.Domain).filter_by(id=domain_id, status=1).first() return domain @@ -37,7 +37,7 @@ def get_domain_by_id(self, domain_id): session.close() def list_by_user(self, user_id): - session = self.make_session() + session = self.session_factory() try: domains = session.query(db.Domain).filter_by(userId=user_id, status=1).all() return domains @@ -45,7 +45,7 @@ def list_by_user(self, user_id): session.close() def list_all(self): - session = self.make_session() + session = self.session_factory() try: domains = session.query(db.Domain).filter_by(status=1).all() return domains @@ -54,7 +54,7 @@ def list_all(self): def register(self, domain_name, user_id): - session = self.make_session() + session = self.session_factory() try: now = datetime.now() domain = db.Domain(userId=user_id, @@ -71,7 +71,7 @@ def register(self, domain_name, user_id): session.close() def renew(self, domain_name): - session = self.make_session() + session = self.session_factory() try: domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() if domain: @@ -84,7 +84,7 @@ def renew(self, domain_name): session.close() def release(self, domain_name): - session = self.make_session() + session = self.session_factory() try: domain = session.query(db.Domain).filter_by(domain=domain_name, status=1).first() if domain: @@ -96,3 +96,14 @@ def release(self, domain_name): session.rollback() finally: session.close() + + def count_domain(self): + session = self.session_factory() + try: + count = session.query(db.Domain).count() + return count + except Exception as e: + logging.error("Error counting domain: %s", e) + return 0 + finally: + session.close() diff --git a/models/glues.py b/models/glues.py index 5227460..9ad6b50 100644 --- a/models/glues.py +++ b/models/glues.py @@ -6,24 +6,24 @@ class Glues: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.session = scoped_session(sessionmaker(bind=self.sql_engine)) + self.session_factory = scoped_session(sessionmaker(bind=self.sql_engine)) def get_record(self, glue_id): - session = self.session() + session = self.session_factory() try: return session.query(db.Glue).filter_by(id=glue_id, status=1).first() finally: session.close() def get_records(self, domain_id): - session = self.session() + session = self.session_factory() try: return session.query(db.Glue).filter_by(domain=domain_id, status=1).all() finally: session.close() def get_record_by_type_value(self, domain_id, subdomain, type_, value): - session = self.session() + session = self.session_factory() try: return session.query(db.Glue).filter_by(domain=domain_id, subdomain=subdomain, @@ -34,7 +34,7 @@ def get_record_by_type_value(self, domain_id, subdomain, type_, value): session.close() def add_record(self, domain_id, subdomain, type_, value, ttl): - session = self.session() + session = self.session_factory() try: new_record = db.Glue( domain=domain_id, @@ -54,7 +54,7 @@ def add_record(self, domain_id, subdomain, type_, value, ttl): session.close() def del_record(self, glue_id): - session = self.session() + session = self.session_factory() try: record_to_delete = session.query(db.Glue).filter_by(id=glue_id).first() if record_to_delete: diff --git a/models/records.py b/models/records.py index 44f08a2..867f2e6 100644 --- a/models/records.py +++ b/models/records.py @@ -6,24 +6,24 @@ class Records: def __init__(self, sql_engine): self.sql_engine = sql_engine - self.session_maker = scoped_session(sessionmaker(bind=self.sql_engine)) + self.session_factory = scoped_session(sessionmaker(bind=self.sql_engine)) def get_record(self, record_id): - session = self.session_maker() + session = self.session_factory() try: return session.query(db.Record).filter_by(id=record_id, status=1).first() finally: session.close() def get_records(self, domain_id): - session = self.session_maker() + session = self.session_factory() try: return session.query(db.Record).filter_by(domain=domain_id, status=1).all() finally: session.close() def get_record_by_type_value(self, domain_id, type_, value): - session = self.session_maker() + session = self.session_factory() try: return session.query(db.Record).filter_by(domain=domain_id, type=type_, @@ -33,7 +33,7 @@ def get_record_by_type_value(self, domain_id, type_, value): session.close() def add_record(self, domain_id, record_type, value, ttl): - session = self.session_maker() + session = self.session_factory() try: record = db.Record(domain=domain_id, type=record_type, @@ -50,7 +50,7 @@ def add_record(self, domain_id, record_type, value, ttl): session.close() def del_record_by_id(self, record_id): - session = self.session_maker() + session = self.session_factory() try: record = session.query(db.Record).filter_by(id=record_id).first() if record: diff --git a/models/users.py b/models/users.py index 3722057..f552164 100644 --- a/models/users.py +++ b/models/users.py @@ -56,3 +56,14 @@ def update_password(self, uid, password): session.rollback() finally: session.close() + + def count_user(self): + session = self.session_factory() + try: + count = session.query(db.User).count() + return count + except Exception as e: + logging.error("Error counting user: %s", e) + return 0 + finally: + session.close() diff --git a/services/auth_service.py b/services/auth_service.py index beccd9a..3fc91d5 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -105,3 +105,6 @@ def verify_email(self, email): return False except EmailNotValidError: return False + + def count_user(self): + return self.users.count_user() diff --git a/services/dns_service.py b/services/dns_service.py index e4abecb..5a679ea 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -186,3 +186,6 @@ def list_domains(self): for domain in domains: result.append(self.__get_domain_info(domain)) return result + + def count_domain(self): + return self.domains.count_domain() diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..a67efe7 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,17 @@ +import json +import requests +from .test_common import get_headers, URL_BASE + +def test_user_count(): + get_headers("0716023") + get_headers("10360874") + get_headers("109550004") + get_headers("109550028") + get_headers("109550032") + + response = requests.get( + URL_BASE + "metrics/", + timeout=10 + ) + num_of_user = json.loads(response.text)['num_of_user'] + assert num_of_user >= 5 From 5c618b51dd5aab7589d780c0217b4cc199ae14b3 Mon Sep 17 00:00:00 2001 From: Lin Lee Date: Sun, 21 Jan 2024 20:06:07 +0800 Subject: [PATCH 86/93] Update main.py --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 13ba0c8..cb1d643 100644 --- a/main.py +++ b/main.py @@ -29,7 +29,7 @@ else: connection_string = ( f"mysql+pymysql://{config.MYSQL_USER}:{config.MYSQL_PSWD}" - f"@{config.MYSQL_HOST}/{config.MYSQL_DB}?wait_timeout=31536000" + f"@{config.MYSQL_HOST}/{config.MYSQL_DB}" ) SQL_ENGINE = create_engine(connection_string) From 7acd69a85cf1b402dd035178ef7e918616831e3e Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 24 Jan 2024 14:00:21 -0500 Subject: [PATCH 87/93] Add support for DNSSEC. --- .pylintrc | 2 +- controllers/__init__.py | 1 + controllers/dnssec.py | 69 ++++++++++++++++++++++++++++ launch_thread.py | 12 ++++- main.py | 7 +-- models/__init__.py | 1 + models/db.py | 13 ++++++ models/ddns.py | 10 +++- models/dnskeys.py | 65 ++++++++++++++++++++++++++ services/dns_service.py | 36 ++++++++++++++- tests/test_add_record.py | 86 +++++++++++++++++++++++++++++++++++ tests/test_controllers.py | 75 ++++++++++++++++++++++++++++++ tests/test_ddns.py | 16 +++++++ tests/test_domain_expired.py | 5 +- tests/test_domain_register.py | 32 ++----------- 15 files changed, 390 insertions(+), 40 deletions(-) create mode 100644 controllers/dnssec.py create mode 100644 models/dnskeys.py create mode 100644 tests/test_add_record.py diff --git a/.pylintrc b/.pylintrc index 98f0c2b..a51bd60 100644 --- a/.pylintrc +++ b/.pylintrc @@ -483,7 +483,7 @@ valid-metaclass-classmethod-first-arg=cls max-args=5 # Maximum number of attributes for a class (see R0902). -max-attributes=7 +max-attributes=9 # Maximum number of boolean expressions in an if statement (see R0916). max-bool-expr=5 diff --git a/controllers/__init__.py b/controllers/__init__.py index 9f6d6f8..07eecef 100644 --- a/controllers/__init__.py +++ b/controllers/__init__.py @@ -2,4 +2,5 @@ from .domains import * from .ddns import * from .glue import * +from .dnssec import * from .metrics import * diff --git a/controllers/dnssec.py b/controllers/dnssec.py new file mode 100644 index 0000000..700cd27 --- /dev/null +++ b/controllers/dnssec.py @@ -0,0 +1,69 @@ +from flask import request, g + +from main import app, authService, dnsService +from services import Operation + +@app.route("/dnssec//records/", methods=['POST']) +def add_dnssec_record(domain): + if not g.user: + return {"msg": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + try: + req = request.json + flag = int(req.get('flag')) + algorithm = int(req.get('algorithm')) + value = req.get('value', "") + ttl = int(req.get('ttl', 5)) + + if flag != 256 or flag != 257: + raise ValueError("Invalid flag value.") + + if not 5 <= ttl <= 86400: + raise ValueError("TTL must be between 5 and 86400.") + + if value.count('\n'): + raise ValueError("Invalid value format.") + + except ValueError as e: + return {"msg": f"Invalid input: {e}"}, 400 + + try: + if not g.user.get('isAdmin', False): + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + dnsService.add_dnssec_key(domain_name, flag, algorithm, value, ttl) + return {"msg": "ok"} + except Exception as e: + return {"msg": str(e)}, 403 + +@app.route("/dnssec//records/", methods=['DELETE']) +def del_dnssec_record(domain): + if not g.user: + return {"msg": "Unauth."}, 401 + + domain_struct = domain.lower().strip('/').split('/') + domain_name = '.'.join(reversed(domain_struct)) + + try: + req = request.json + flag = int(req.get('flag')) + if flag != 256 or flag != 257: + raise ValueError("Invalid flag value.") + algorithm = int(req.get('algorithm')) + value = req.get('value', "") + + if value.count('\n'): + raise ValueError("Invalid value format.") + + except ValueError as e: + return {"msg": f"Invalid input: {e}"}, 400 + + try: + if not g.user['isAdmin']: + authService.authorize_action(g.user['uid'], Operation.MODIFY, domain_name) + dnsService.del_dnssec_key(domain_name, flag, algorithm, value) + return {"msg": "ok"} + except Exception as e: + return {"msg": str(e)}, 403 diff --git a/launch_thread.py b/launch_thread.py index a6f1456..79fd19c 100644 --- a/launch_thread.py +++ b/launch_thread.py @@ -4,7 +4,7 @@ from sqlalchemy import create_engine import config -from models import Users, Domains, Records, Glues, DDNS, db +from models import Users, Domains, Records, Glues, Dnskeys, DDNS, db from services import AuthService, DNSService def recycle(local_dns_service): @@ -34,9 +34,17 @@ def recycle(local_dns_service): domains = Domains(SQL_ENGINE) records = Records(SQL_ENGINE) glues = Glues(SQL_ENGINE) +dnskeys = Dnskeys(SQL_ENGINE) auth_service = AuthService(logging, config.JWT_SECRET, users, domains) -dns_service = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) +dns_service = DNSService(logging, + users, + domains, + records, + glues, + dnskeys, + ddns, + config.HOST_DOMAINS) if __name__ == "__main__": while True: diff --git a/main.py b/main.py index cb1d643..590a00a 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ from sqlalchemy import create_engine import config -from models import Users, Domains, Records, Glues, DDNS, Elastic, db +from models import Users, Domains, Records, Glues, Dnskeys, DDNS, Elastic, db from services import AuthService, DNSService, MailService, Oauth env_test = os.getenv('TEST') @@ -39,6 +39,7 @@ domains = Domains(SQL_ENGINE) records = Records(SQL_ENGINE) glues = Glues(SQL_ENGINE) +dnskeys = Dnskeys(SQL_ENGINE) nycu_oauth = Oauth(redirect_uri = config.NYCU_OAUTH_RURL, client_id = config.NYCU_OAUTH_ID, @@ -46,7 +47,7 @@ elastic = Elastic(config.ELASTICSERVER, config.ELASTICUSER, config.ELASTICPASS) authService = AuthService(logging, config.JWT_SECRET, users, domains) -dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) +dnsService = DNSService(logging, users, domains, records, glues, dnskeys, ddns, config.HOST_DOMAINS) mailService = MailService(logging, config.SMTP_SERVER, config.SMTP_PORT, config.SMTP_USER, config.SMTP_PASS, config.SMTP_FROM) -from controllers import auth, domains, ddns, glue, metrics # pylint: disable=all +from controllers import auth, domains, ddns, glue, dnssec, metrics # pylint: disable=all diff --git a/models/__init__.py b/models/__init__.py index 5fa50fb..b0d1b45 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -4,4 +4,5 @@ from .domains import * from .records import * from .glues import * +from .dnskeys import * from .elastic import * diff --git a/models/db.py b/models/db.py index cfb8d63..0817978 100644 --- a/models/db.py +++ b/models/db.py @@ -53,3 +53,16 @@ class Glue(Base): regDate = Column(DateTime) expDate = Column(DateTime) status = Column(BOOLEAN, default=True) + +class Dnskey(Base): + __tablename__ = 'dnskeys' + + id = Column(Integer, primary_key=True, autoincrement=True) + domain = Column(Integer, ForeignKey('domains.id'), nullable=False) + ttl = Column(Integer, nullable=False) + flag = Column(Integer, nullable=False) + algorithm = Column(Integer, nullable=False) + value = Column(String(512)) + regDate = Column(DateTime) + expDate = Column(DateTime) + status = Column(BOOLEAN, default=True) diff --git a/models/ddns.py b/models/ddns.py index 89f8d39..8d4e539 100644 --- a/models/ddns.py +++ b/models/ddns.py @@ -33,7 +33,7 @@ def __write(self): self.nsupdate.stdin.write((cmd + "\n").encode()) diff = 1 - logging.debug("executing command: %s", cmd) + print(f"executing command: {cmd}") if diff and self.nsupdate.poll() is None: diff = 0 @@ -72,4 +72,10 @@ def del_record(self, domain, rectype, value): if rectype == "MX": value = f"10 {value}" self.queue.put(f"update delete {domain} {rectype} {value}") - print(f"update delete {domain} {rectype} {value}") + + def add_dnskey_record(self, domain, rectype, algorithm, value, ttl = 5): + self.queue.put(f"update add {domain} {ttl} DNSKEY {rectype} 3 {algorithm} {value}") + + def del_dnskey_record(self, domain, rectype, algorithm, value): + self.queue.put(f"update delete {domain} DNSKEY {rectype} 3 {algorithm} {value}") + diff --git a/models/dnskeys.py b/models/dnskeys.py new file mode 100644 index 0000000..f2960dd --- /dev/null +++ b/models/dnskeys.py @@ -0,0 +1,65 @@ +from datetime import datetime +from sqlalchemy.orm import sessionmaker, scoped_session + +from . import db +class Dnskeys: + def __init__(self, sql_engine): + self.sql_engine = sql_engine + self.session_factory = scoped_session(sessionmaker(bind=self.sql_engine)) + + def add_dnskey_record(self, domain_id, flag, algorithm, value, ttl): + session = self.session_factory() + try: + new_dnskey = db.Dnskey( + domain=domain_id, + flag=flag, + algorithm=algorithm, + value=value, + ttl=ttl, + regDate=datetime.now(), + status=1 + ) + session.add(new_dnskey) + session.commit() + return new_dnskey + except Exception as e: + session.rollback() + raise e + finally: + session.close() + + def del_dnskey_record(self, dnskey_id): + session = self.session_factory() + try: + dnskey_to_remove = session.query(db.Dnskey).filter_by(id=dnskey_id).first() + if dnskey_to_remove: + dnskey_to_remove.expDate = datetime.now() + dnskey_to_remove.status = 0 + session.commit() + return True + return False + except Exception as e: + session.rollback() + raise e + finally: + session.close() + + def get_records(self, domain_id): + session = self.session_factory() + try: + return session.query(db.Dnskey).filter_by(domain=domain_id, status=1).all() + finally: + session.close() + + def get_dnskey_by_value(self, domain_id, flag, algorithm, value): + session = self.session_factory() + try: + return session.query(db.Dnskey).filter_by( + domain=domain_id, + flag=flag, + algorithm=algorithm, + value=value, + status=1 + ).first() + finally: + session.close() diff --git a/services/dns_service.py b/services/dns_service.py index 5a679ea..ed603e7 100644 --- a/services/dns_service.py +++ b/services/dns_service.py @@ -20,12 +20,13 @@ def __repr__(self): return f"{self.typ}: {self.msg}" class DNSService(): - def __init__(self, logger, users, domains, records, glues, ddns, host_domains): + def __init__(self, logger, users, domains, records, glues, dnskeys, ddns, host_domains): self.logger = logger self.users = users self.domains = domains self.records = records self.glues = glues + self.dnskeys = dnskeys self.ddns = ddns self.host_domains = host_domains @@ -37,8 +38,10 @@ def __get_domain_info(self, domain): domain_info['domain'] = domain.domain domain_info['records'] = [] domain_info['glues'] = [] + domain_info['dnskeys'] = [] records = self.records.get_records(domain.id) glues = self.glues.get_records(domain.id) + dnskeys = self.dnskeys.get_records(domain.id) for record in records: domain_info['records'].append((record.id, record.type, @@ -50,6 +53,12 @@ def __get_domain_info(self, domain): record.type, record.value, record.ttl)) + for record in dnskeys: + domain_info['dnskeys'].append((record.id, + record.flag, + record.algorithm, + record.value, + record.ttl)) return domain_info def check_domain(self, domain_name): @@ -125,10 +134,13 @@ def release_domain(self, domain_name): raise DNSError(DNSErrors.NXDOMAIN, "This domain is not registered.") records = self.records.get_records(domain.id) glues = self.glues.get_records(domain.id) + dnskeys = self.dnskeys.get_records(domain.id) for record in records: self.del_record_by_id(record.id) for record in glues: self.del_glue_record(domain.domain, record.subdomain, record.type, record.value) + for record in dnskeys: + self.del_dnssec_key(domain_name, record.flag, record.algorithm, record.value) self.domains.release(domain_name) def add_record(self, domain_name, type_, value, ttl): @@ -180,6 +192,28 @@ def del_glue_record(self, domain_name, subdomain, type_, value): self.glues.del_record(glue_record.id) self.ddns.del_record(real_domain, type_, value) + def add_dnssec_key(self, domain_name, flag, algorithm, value, ttl): + domain = self.domains.get_domain(domain_name) + self.dnskeys.add_dnskey_record( + domain.id, + flag, + algorithm, + value, + ttl + ) + self.ddns.add_dnskey_record(domain_name, flag, algorithm, value, ttl) + + def del_dnssec_key(self, domain_name, flag, algorithm, value): + domain = self.domains.get_domain(domain_name) + dnssec_key = self.dnskeys.get_dnskey_by_value( + domain.id, + flag, + algorithm, + value + ) + self.dnskeys.del_dnskey_record(dnssec_key.id) + self.ddns.del_dnskey_record(domain_name, flag, algorithm, value) + def list_domains(self): domains = self.domains.list_all() result = [] diff --git a/tests/test_add_record.py b/tests/test_add_record.py new file mode 100644 index 0000000..2ed236e --- /dev/null +++ b/tests/test_add_record.py @@ -0,0 +1,86 @@ +import time +import logging +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +import pydig + +from models import Domains, Records, Users, Glues, Dnskeys, db, DDNS +from services import DNSService +import config + + +ddns = DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") + +resolver = pydig.Resolver( + executable='/usr/bin/dig', + nameservers=[ + '172.21.21.3' + ], +) + +sql_engine = create_engine('sqlite:///:memory:') +db.Base.metadata.create_all(sql_engine) +Session = sessionmaker(bind=sql_engine) +session = Session() + +users = Users(sql_engine) +domains = Domains(sql_engine) +records = Records(sql_engine) +glues = Glues(sql_engine) +dnskeys = Dnskeys(sql_engine) + +dnsService = DNSService(logging, users, domains, records, glues, dnskeys, ddns, config.HOST_DOMAINS) + +testdata = [("test-reg.nycu-dev.me", 'A', "140.113.89.64", 5), + ("test-reg.nycu-dev.me", 'A', "140.113.64.89", 5)] +answer = {"140.113.89.64", "140.113.64.89"} + +def test_duplicated_record(): + dnsService.register_domain("109550028", "test-add-dup-rec.nycu-dev.me") + dnsService.add_record("test-add-dup-rec.nycu-dev.me", 'A', "140.113.64.89", 5) + try: + dnsService.add_record("test-add-dup-rec.nycu-dev.me", 'A', "140.113.64.89", 5) + assert 0 + except Exception: + assert 1 + dnsService.release_domain("test-add-dup-rec.nycu-dev.me") + +def test_glue_record(): + dnsService.register_domain("109550028", "test-glue.nycu-dev.me") + + dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) + time.sleep(5) + assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == {"1.1.1.1"} + + dnsService.del_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") + time.sleep(5) + assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() + + # check if glue record is be removed after domain released + dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) + dnsService.release_domain("test-glue.nycu-dev.me") + time.sleep(5) + assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() + +def test_add_dnskey_record(): + dnsService.register_domain("109550028", "test-dnskey.nycu-dev.me") + dnsService.add_dnssec_key( + "test-dnskey.nycu-dev.me", + "257", + "13", + "oGPBfdLt+oJa6pAnDHtNcZ61d5MWfeocmxdkBI7YuS8D5MOMxLtc7Kyr " + + "ItibqhKrrBh4m73uy4N6fRhf2e5Bug==", + 5) + time.sleep(5) + assert set(resolver.query("test-dnskey.nycu-dev.me", 'dnskey')) == { + "257 3 13 oGPBfdLt+oJa6pAnDHtNcZ61d5MWfeocmxdkBI7YuS8D5MOMxLtc7Kyr " + + "ItibqhKrrBh4m73uy4N6fRhf2e5Bug==" + } + dnsService.del_dnssec_key( + "test-dnskey.nycu-dev.me", + "257", + "13", + "oGPBfdLt+oJa6pAnDHtNcZ61d5MWfeocmxdkBI7YuS8D5MOMxLtc7Kyr " + + "ItibqhKrrBh4m73uy4N6fRhf2e5Bug==") + time.sleep(5) + assert set(resolver.query("test-dnskey.nycu-dev.me", 'dnskey')) == set() diff --git a/tests/test_controllers.py b/tests/test_controllers.py index bf370b1..2f51ea1 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -216,3 +216,78 @@ def test_ttl(ttl, answer): timeout=10 ) assert response.status_code == 200 + +def test_dnssec(): + headers = get_headers("10360874") + + # Register domains + response = requests.post( + URL_BASE + "domains/me/nycu-dev/test-dnssec", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + # Add KSK + response = requests.post( + URL_BASE + "dnssec/me/nycu-dev/test-dnssec/records/", + json = { + "flag": 257, + "algorithm": 13, + 'value': ( + "oGPBfdLt+oJa6pAnDHtNcZ61d5MWfeocmxdkBI7YuS8D5MOMxLtc7Kyr "+ + "ItibqhKrrBh4m73uy4N6fRhf2e5Bug==" + ), + 'ttl': 5 + }, + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + time.sleep(5) + assert set(resolver.query("test-dnssec.nycu-dev.me", 'dnskey')) == { + "257 3 13 oGPBfdLt+oJa6pAnDHtNcZ61d5MWfeocmxdkBI7YuS8D5MOMxLtc7Kyr " + + "ItibqhKrrBh4m73uy4N6fRhf2e5Bug==" + } + # Del KSK + response = requests.delete( + URL_BASE + "dnssec/me/nycu-dev/test-dnssec/records/", + json = { + "flag": 257, + "algorithm": 13, + 'value': ( + "oGPBfdLt+oJa6pAnDHtNcZ61d5MWfeocmxdkBI7YuS8D5MOMxLtc7Kyr "+ + "ItibqhKrrBh4m73uy4N6fRhf2e5Bug==" + ) + }, + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + time.sleep(5) + assert set(resolver.query("test-dnssec.nycu-dev.me", 'dnskey')) == set() + # Test recycling + response = requests.post( + URL_BASE + "dnssec/me/nycu-dev/test-dnssec/records/", + json = { + "flag": 257, + "algorithm": 13, + 'value': ( + "oGPBfdLt+oJa6pAnDHtNcZ61d5MWfeocmxdkBI7YuS8D5MOMxLtc7Kyr "+ + "ItibqhKrrBh4m73uy4N6fRhf2e5Bug==" + ), + 'ttl': 5 + }, + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + time.sleep(5) + assert set(resolver.query("test-dnssec.nycu-dev.me", 'dnskey')) != set() + response = requests.delete( + URL_BASE + "domains/me/nycu-dev/test-dnssec", + headers = headers, + timeout=10 + ) + assert response.status_code == 200 + time.sleep(5) + assert set(resolver.query("test-dnssec.nycu-dev.me", 'dnskey')) == set() diff --git a/tests/test_ddns.py b/tests/test_ddns.py index c030174..391a08a 100644 --- a/tests/test_ddns.py +++ b/tests/test_ddns.py @@ -16,6 +16,11 @@ ("test2-ddns.nycu-dev.me", 'A', "140.113.69.69", 86400), ] testdata_mx = ("test-ddns.nycu-dev.me", 'MX', "test-ddns.nycu-dev.me", 5) +testdata_dnskey = ("test-ddns-dnskey.nycu-dev.me", + '257', + "13", + "oGPBfdLt+oJa6pAnDHtNcZ61d5MWfeocmxdkBI7YuS8D5MOMxLtc7Kyr " + + "ItibqhKrrBh4m73uy4N6fRhf2e5Bug==") def test_add_a_record(): domains = {} @@ -40,3 +45,14 @@ def test_add_mx_record(): ddns.del_record(*testdata_mx[:-1]) time.sleep(5) assert set(resolver.query(testdata_mx[0], 'MX')) == set() + +def test_add_dnskey_record(): + ddns.add_dnskey_record(*testdata_dnskey) + time.sleep(5) + assert set(resolver.query(testdata_dnskey[0], 'dnskey')) == { + "257 3 13 oGPBfdLt+oJa6pAnDHtNcZ61d5MWfeocmxdkBI7YuS8D5MOMxLtc7Kyr " + + "ItibqhKrrBh4m73uy4N6fRhf2e5Bug==" + } + ddns.del_dnskey_record(*testdata_dnskey) + time.sleep(5) + assert set(resolver.query(testdata_dnskey[0], 'dnskey')) == set() diff --git a/tests/test_domain_expired.py b/tests/test_domain_expired.py index 2f1af16..4df9247 100644 --- a/tests/test_domain_expired.py +++ b/tests/test_domain_expired.py @@ -4,7 +4,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from models import Domains, Records, Users, Glues, db, DDNS +from models import Domains, Records, Users, Glues, Dnskeys, db, DDNS from services import DNSService import config from launch_thread import recycle @@ -20,8 +20,9 @@ domains = Domains(sql_engine) records = Records(sql_engine) glues = Glues(sql_engine) +dnskeys = Dnskeys(sql_engine) -dnsService = DNSService(logging, users, domains, glues, records, ddns, config.HOST_DOMAINS) +dnsService = DNSService(logging, users, domains, records, glues, dnskeys, ddns, config.HOST_DOMAINS) def test_domain_expire(): exp_date = datetime.now() + timedelta(seconds=5) diff --git a/tests/test_domain_register.py b/tests/test_domain_register.py index 8fc9cf3..ecc99e7 100644 --- a/tests/test_domain_register.py +++ b/tests/test_domain_register.py @@ -4,7 +4,7 @@ from sqlalchemy.orm import sessionmaker import pydig -from models import Domains, Records, Users, Glues, db, DDNS +from models import Domains, Records, Users, Glues, Dnskeys, db, DDNS from services import DNSService import config @@ -27,8 +27,9 @@ domains = Domains(sql_engine) records = Records(sql_engine) glues = Glues(sql_engine) +dnskeys = Dnskeys(sql_engine) -dnsService = DNSService(logging, users, domains, records, glues, ddns, config.HOST_DOMAINS) +dnsService = DNSService(logging, users, domains, records, glues, dnskeys, ddns, config.HOST_DOMAINS) testdata = [("test-reg.nycu-dev.me", 'A', "140.113.89.64", 5), ("test-reg.nycu-dev.me", 'A', "140.113.64.89", 5)] @@ -72,30 +73,3 @@ def test_unhost_register(): assert 0 except Exception: assert 1 - -def test_duplicated_record(): - dnsService.register_domain("109550028", "test-add-dup-rec.nycu-dev.me") - dnsService.add_record("test-add-dup-rec.nycu-dev.me", 'A', "140.113.64.89", 5) - try: - dnsService.add_record("test-add-dup-rec.nycu-dev.me", 'A', "140.113.64.89", 5) - assert 0 - except Exception: - assert 1 - dnsService.release_domain("test-add-dup-rec.nycu-dev.me") - -def test_glue_record(): - dnsService.register_domain("109550028", "test-glue.nycu-dev.me") - - dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) - time.sleep(5) - assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == {"1.1.1.1"} - - dnsService.del_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1") - time.sleep(5) - assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() - - # check if glue record is be removed after domain released - dnsService.add_glue_record("test-glue.nycu-dev.me", "abc", "A", "1.1.1.1", 5) - dnsService.release_domain("test-glue.nycu-dev.me") - time.sleep(5) - assert set(resolver.query("abc.test-glue.nycu-dev.me", 'A')) == set() From 6561df992589f5db6c5384ace5f55a42fe00943b Mon Sep 17 00:00:00 2001 From: LeeLin2602 Date: Wed, 24 Jan 2024 14:36:40 -0500 Subject: [PATCH 88/93] fix tiny bug: misuing `or` and `and` --- controllers/dnssec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controllers/dnssec.py b/controllers/dnssec.py index 700cd27..23d6334 100644 --- a/controllers/dnssec.py +++ b/controllers/dnssec.py @@ -18,9 +18,9 @@ def add_dnssec_record(domain): value = req.get('value', "") ttl = int(req.get('ttl', 5)) - if flag != 256 or flag != 257: + if flag not in (256, 257): raise ValueError("Invalid flag value.") - + if not 5 <= ttl <= 86400: raise ValueError("TTL must be between 5 and 86400.") @@ -49,7 +49,7 @@ def del_dnssec_record(domain): try: req = request.json flag = int(req.get('flag')) - if flag != 256 or flag != 257: + if flag not in (256, 257): raise ValueError("Invalid flag value.") algorithm = int(req.get('algorithm')) value = req.get('value', "") From f289ac1088c5da924bc3ecc7322ac06876dbbec5 Mon Sep 17 00:00:00 2001 From: Akhilesh <96287396+Akhilesh1004@users.noreply.github.com> Date: Sat, 27 Jan 2024 14:05:02 +0800 Subject: [PATCH 89/93] test_sql_trigger --- tests/test_sql_trigger.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/test_sql_trigger.py diff --git a/tests/test_sql_trigger.py b/tests/test_sql_trigger.py new file mode 100644 index 0000000..f2ffcfc --- /dev/null +++ b/tests/test_sql_trigger.py @@ -0,0 +1,9 @@ + +from sqlalchemy import create_engine + +from models import db + + +def test_sql_trigger(): + sql_engine = create_engine('sqlite:///:memory:') + db.Base.metadata.create_all(sql_engine) From ab2f8b6cc4041b72d22a424196a560dc260aadb1 Mon Sep 17 00:00:00 2001 From: Akhilesh <96287396+Akhilesh1004@users.noreply.github.com> Date: Sat, 27 Jan 2024 17:10:08 +0800 Subject: [PATCH 90/93] change --- models/domains.py | 1 + tests/test_sql_trigger.py | 48 +++++++++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/models/domains.py b/models/domains.py index 4ffc8e3..4a7e9e6 100644 --- a/models/domains.py +++ b/models/domains.py @@ -67,6 +67,7 @@ def register(self, domain_name, user_id): except Exception as e: logging.error("Error registering domain: %s", e) session.rollback() + raise e finally: session.close() diff --git a/tests/test_sql_trigger.py b/tests/test_sql_trigger.py index f2ffcfc..e436fc2 100644 --- a/tests/test_sql_trigger.py +++ b/tests/test_sql_trigger.py @@ -1,9 +1,49 @@ - +import time +import logging from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +import pydig + +from models import Domains, Records, Users, Glues, Dnskeys, db, DDNS +from services import DNSService +import config + + +ddns = DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") -from models import db +resolver = pydig.Resolver( + executable='/usr/bin/dig', + nameservers=[ + '172.21.21.3' + ], +) +sql_engine = create_engine('sqlite:///:memory:') +db.Base.metadata.create_all(sql_engine) +Session = sessionmaker(bind=sql_engine) +session = Session() + +users = Users(sql_engine) +domains = Domains(sql_engine) +records = Records(sql_engine) +glues = Glues(sql_engine) +dnskeys = Dnskeys(sql_engine) + +dnsService = DNSService(logging, users, domains, records, glues, dnskeys, ddns, config.HOST_DOMAINS) + +testdata = [("test-reg.nycu-dev.me", 'A', "140.113.89.64", 5), + ("test-reg.nycu-dev.me", 'A', "140.113.64.89", 5)] +answer = {"140.113.89.64", "140.113.64.89"} def test_sql_trigger(): - sql_engine = create_engine('sqlite:///:memory:') - db.Base.metadata.create_all(sql_engine) + dnsService.register_domain("109550028", "test-reg.nycu-dev.me") + for testcase in testdata: + dnsService.add_record(*testcase) + time.sleep(10) + assert set(resolver.query("test-reg.nycu-dev.me", 'A')) == answer + dnsService.release_domain("test-reg.nycu-dev.me") + dnsService.register_domain("109550028", "test-reg.nycu-dev.me") + time.sleep(10) + assert set(resolver.query("test-reg.nycu-dev.me", 'A')) == set() + + From 46108485fd321a440e8e53b4c65c524e4e3a917e Mon Sep 17 00:00:00 2001 From: Akhilesh <96287396+Akhilesh1004@users.noreply.github.com> Date: Sun, 28 Jan 2024 14:43:18 +0800 Subject: [PATCH 91/93] test_sql_trigger --- tests/test_sql_trigger.py | 84 +++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/test_sql_trigger.py b/tests/test_sql_trigger.py index e436fc2..382dacc 100644 --- a/tests/test_sql_trigger.py +++ b/tests/test_sql_trigger.py @@ -1,49 +1,49 @@ -import time -import logging -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -import pydig +from datetime import datetime, timedelta +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker, scoped_session +from models import db -from models import Domains, Records, Users, Glues, Dnskeys, db, DDNS -from services import DNSService -import config - - -ddns = DDNS(logging, "/etc/ddnskey.conf", "172.21.21.3", "nycu-dev.me") - -resolver = pydig.Resolver( - executable='/usr/bin/dig', - nameservers=[ - '172.21.21.3' - ], -) sql_engine = create_engine('sqlite:///:memory:') -db.Base.metadata.create_all(sql_engine) -Session = sessionmaker(bind=sql_engine) -session = Session() - -users = Users(sql_engine) -domains = Domains(sql_engine) -records = Records(sql_engine) -glues = Glues(sql_engine) -dnskeys = Dnskeys(sql_engine) -dnsService = DNSService(logging, users, domains, records, glues, dnskeys, ddns, config.HOST_DOMAINS) +db.Base.metadata.create_all(sql_engine) -testdata = [("test-reg.nycu-dev.me", 'A', "140.113.89.64", 5), - ("test-reg.nycu-dev.me", 'A', "140.113.64.89", 5)] -answer = {"140.113.89.64", "140.113.64.89"} +trigger_sql = """ +CREATE TRIGGER before_insert_domains +BEFORE INSERT ON domains FOR EACH ROW +WHEN NEW.status = 1 AND ( + SELECT COUNT(*) + FROM domains + WHERE domain = NEW.domain AND status = 1 +) > 0 +BEGIN + SELECT RAISE(ABORT, 'This domain has been registered'); +END; +""" + +# 执行 Trigger SQL 语句 +with sql_engine.connect() as connection: + connection.execute(text(trigger_sql)) + +# Use scoped_session for thread safety +Session = scoped_session(sessionmaker(bind=sql_engine)) + +def insert_domain(): + session = Session() + now = datetime.now() + domain = db.Domain(userId="109550028", + domain="test-reg.nycu-dev.me", + regDate=now, + expDate=now + timedelta(days=90), + status=1) + session.add(domain) + session.commit() + session.close() def test_sql_trigger(): - dnsService.register_domain("109550028", "test-reg.nycu-dev.me") - for testcase in testdata: - dnsService.add_record(*testcase) - time.sleep(10) - assert set(resolver.query("test-reg.nycu-dev.me", 'A')) == answer - dnsService.release_domain("test-reg.nycu-dev.me") - dnsService.register_domain("109550028", "test-reg.nycu-dev.me") - time.sleep(10) - assert set(resolver.query("test-reg.nycu-dev.me", 'A')) == set() - - + insert_domain() + try: + insert_domain() + assert 0 + except Exception: + assert 1 From f37f77b5580173d2827cbea309333ed0ad11f031 Mon Sep 17 00:00:00 2001 From: Akhilesh <96287396+Akhilesh1004@users.noreply.github.com> Date: Sun, 28 Jan 2024 14:54:10 +0800 Subject: [PATCH 92/93] test_sql_trigger --- tests/test_sql_trigger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sql_trigger.py b/tests/test_sql_trigger.py index 382dacc..ae2a78c 100644 --- a/tests/test_sql_trigger.py +++ b/tests/test_sql_trigger.py @@ -8,7 +8,7 @@ db.Base.metadata.create_all(sql_engine) -trigger_sql = """ +TRIGGER_SQL = """ CREATE TRIGGER before_insert_domains BEFORE INSERT ON domains FOR EACH ROW WHEN NEW.status = 1 AND ( @@ -23,7 +23,7 @@ # 执行 Trigger SQL 语句 with sql_engine.connect() as connection: - connection.execute(text(trigger_sql)) + connection.execute(text(TRIGGER_SQL)) # Use scoped_session for thread safety Session = scoped_session(sessionmaker(bind=sql_engine)) From deb3f080a5ccc770c602d75e2ac2f13deda8e290 Mon Sep 17 00:00:00 2001 From: Akhilesh <96287396+Akhilesh1004@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:17:03 +0800 Subject: [PATCH 93/93] change comment --- tests/test_sql_trigger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sql_trigger.py b/tests/test_sql_trigger.py index ae2a78c..24fd896 100644 --- a/tests/test_sql_trigger.py +++ b/tests/test_sql_trigger.py @@ -21,7 +21,7 @@ END; """ -# 执行 Trigger SQL 语句 +# execute Trigger SQL with sql_engine.connect() as connection: connection.execute(text(TRIGGER_SQL))