diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c47553c7..947cd2b03 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -88,7 +88,7 @@ jobs: working-directory: rpi_data run: | curl --insecure -X POST -H "Content-Type: multipart/form-data" \ - -F "isPubliclyVisible=on" -F "file=@$FILE_NAME" \ + -F "is_publicly_visible=on" -F "file=@$FILE_NAME" \ $AUTH_ARGS $API_ENDPOINT env: FILE_NAME: fall-2021.csv diff --git a/.github/workflows/prod-update.yaml b/.github/workflows/prod-update.yaml index 88a89e022..25fda114a 100644 --- a/.github/workflows/prod-update.yaml +++ b/.github/workflows/prod-update.yaml @@ -38,7 +38,7 @@ jobs: - name: Update prod run: | - curl -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=@$FILE_NAME" -A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" $AUTH_ARGS $API_ENDPOINT + curl -X POST -H "Content-Type: multipart/form-data" -F "is_publicly_visible=on" -F "file=@$FILE_NAME" -A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" $AUTH_ARGS $API_ENDPOINT env: FILE_NAME: ${{ matrix.semester }}.csv AUTH_ARGS: ${{ secrets.AUTH_ARGS }} diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..6272dc421 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,653 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS, + wsgi.py + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.9 + +# Discover python modules and packages in the file system subtree. +recursive=yes + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots=./src/api, + ./rpi_data/modules + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + use-implicit-booleaness-not-comparison-to-string, + use-implicit-booleaness-not-comparison-to-zero, + missing-module-docstring, + missing-function-docstring, + missing-class-docstring, + consider-using-enumerate, + unidiomatic-typecheck, + broad-exception-caught, + wildcard-import, + unused-wildcard-import, + consider-using-with, + duplicate-code, + broad-exception-raised, + redefined-outer-name + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are: text, parseable, colorized, +# json2 (improved json format), json (old json format) and msvs (visual +# studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: en_BW (hunspell), en_AU +# (hunspell), en_BZ (hunspell), en_GB (hunspell), en_JM (hunspell), en_DK +# (hunspell), en_HK (hunspell), en_GH (hunspell), en_US (hunspell), en_ZA +# (hunspell), en_ZW (hunspell), en_SG (hunspell), en_NZ (hunspell), en_BS +# (hunspell), en_AG (hunspell), en_PH (hunspell), en_IE (hunspell), en_NA +# (hunspell), en_TT (hunspell), en_IN (hunspell), en_NG (hunspell), en_CA +# (hunspell). +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/conftest.py b/conftest.py index d8f4b6246..8662adb2e 100644 --- a/conftest.py +++ b/conftest.py @@ -6,8 +6,8 @@ from src.api.db.classinfo import ClassInfo from src.api.db.courses import Courses from src.api.db.admin import Admin -from src.api.db.semester_info import semester_info as SemesterInfo -from src.api.db.semester_date_mapping import semester_date_mapping as SemesterDateMapping +from src.api.db.semester_info import SemesterInfo as SemesterInfo +from src.api.db.semester_date_mapping import SemesterDateMapping as SemesterDateMapping # from src.api.db.student_course_selection import student_course_selection # from src.api.db.user import User from rpi_data.modules.add_school_column import SchoolDepartmentMapping, SCHOOL_DEPARTMENT_MAPPING_YAML_FILENAME diff --git a/ops/scripts/configure.sh b/ops/scripts/configure.sh index eed1ee241..41aee20eb 100644 --- a/ops/scripts/configure.sh +++ b/ops/scripts/configure.sh @@ -44,7 +44,7 @@ function load_semester() { --request POST \ 'http://localhost/api/bulkCourseUpload' \ --form "file=@$1" \ - --form 'isPubliclyVisible=true' \ + --form 'is_publicly_visible=true' \ --max-time 60 \ -v } diff --git a/rpi_data/modules/add_school_column.py b/rpi_data/modules/add_school_column.py index 5144a34d3..7a4330d5a 100644 --- a/rpi_data/modules/add_school_column.py +++ b/rpi_data/modules/add_school_column.py @@ -1,61 +1,63 @@ +from typing import List, Dict + import yaml import pandas as pd -from typing import List, Dict - # File copied from YACS SCHOOL_DEPARTMENT_MAPPING_YAML_FILENAME = "school-department-mapping.yaml" - + class SchoolDepartmentMapping: - def __init__(self, mapping: Dict[str, str], schools: List[str]): - """ :param: mapping - dict that maps department shortname -> school longname - e.g. mapping.get('CSCI') == 'Science' - mapping.get('COGS') == 'Humanities, Arts and Social Sciences' - :param: schools - list of schools - """ - self.mapping = mapping - self.schools = schools - - def get(self, key: str, default = 'Other') -> str: - return self.mapping.get(key, default) - - @classmethod - def parse_yaml(cls, path: str) -> 'SchoolDepartmentMapping': - data = None - - with open(path) as f: - data = yaml.safe_load(f.read()) - - - # data is a dict with the following form - # { - # 'schools': { - # 'longname': str ('Humanities, Arts and Social Sciences'), - # 'subjects': { - # 'shortname': str ('ARTS'), - # 'longname: str ('Arts') - # }[] - # }[] - # } - - - mapping: Dict[str, str] = {} - - for school in data['schools']: - school_longname = school['longname'] - - for subject in school['subjects']: - subject_shortname = subject['shortname'] - - mapping[subject_shortname] = school_longname - - schools = [school['longname'] for school in data['schools']] - - return cls(mapping, schools) - -def add_school_column(df: pd.DataFrame, school_department_mapping_path = SCHOOL_DEPARTMENT_MAPPING_YAML_FILENAME) -> pd.DataFrame: - school_department_mapping = SchoolDepartmentMapping.parse_yaml(school_department_mapping_path) - - df['school'] = df['course_department'].apply(school_department_mapping.get) - - return df \ No newline at end of file + def __init__(self, mapping: Dict[str, str], schools: List[str]): + """ + :param: mapping - dict that maps department shortname -> school longname + e.g. mapping.get('CSCI') == 'Science' + mapping.get('COGS') == 'Humanities, Arts and Social Sciences' + :param: schools - list of schools + """ + self.mapping = mapping + self.schools = schools + + def get(self, key: str, default = 'Other') -> str: + return self.mapping.get(key, default) + + @classmethod + def parse_yaml(cls, path: str) -> 'SchoolDepartmentMapping': + data = None + + with open(path, encoding="utf-8") as f: + data = yaml.safe_load(f.read()) + + + # data is a dict with the following form + # { + # 'schools': { + # 'longname': str ('Humanities, Arts and Social Sciences'), + # 'subjects': { + # 'shortname': str ('ARTS'), + # 'longname: str ('Arts') + # }[] + # }[] + # } + + + mapping: Dict[str, str] = {} + + for school in data['schools']: + school_longname = school['longname'] + + for subject in school['subjects']: + subject_shortname = subject['shortname'] + + mapping[subject_shortname] = school_longname + + schools = [school['longname'] for school in data['schools']] + + return cls(mapping, schools) + +def add_school_column(df: pd.DataFrame,school_department_mapping_path + = SCHOOL_DEPARTMENT_MAPPING_YAML_FILENAME) -> pd.DataFrame: + school_department_mapping = SchoolDepartmentMapping.parse_yaml(school_department_mapping_path) + + df['school'] = df['course_department'].apply(school_department_mapping.get) + + return df diff --git a/rpi_data/modules/build_semester.py b/rpi_data/modules/build_semester.py index 3089901dd..9127492fa 100644 --- a/rpi_data/modules/build_semester.py +++ b/rpi_data/modules/build_semester.py @@ -1,7 +1,10 @@ import os if __name__ == '__main__': - os.system("SOURCE_URL=https://sis.rpi.edu/reg/zs20230501.htm DEST=out1.csv HEADERS=True python rpi-parse.py") - os.system("SOURCE_URL=https://sis.rpi.edu/reg/zs20230502.htm DEST=out2.csv HEADERS=False python rpi-parse.py") - os.system("SOURCE_URL=https://sis.rpi.edu/reg/zs20230503.htm DEST=out3.csv HEADERS=False python rpi-parse.py") - os.system("cat out1.csv out2.csv out3.csv > out.csv") \ No newline at end of file + os.system("SOURCE_URL=https://sis.rpi.edu/reg/zs20230501.htm" + + " DEST=out1.csv HEADERS=True python rpi-parse.py") + os.system("SOURCE_URL=https://sis.rpi.edu/reg/zs20230502.htm" + + "DEST=out2.csv HEADERS=False python rpi-parse.py") + os.system("SOURCE_URL=https://sis.rpi.edu/reg/zs20230503.htm" + + "DEST=out3.csv HEADERS=False python rpi-parse.py") + os.system("cat out1.csv out2.csv out3.csv > out.csv") diff --git a/rpi_data/modules/course.py b/rpi_data/modules/course.py index 5d6036b9c..dcaff9674 100644 --- a/rpi_data/modules/course.py +++ b/rpi_data/modules/course.py @@ -1,7 +1,5 @@ -import time -import pdb import copy -from typing import overload + class Course: name:str credits:int @@ -17,21 +15,22 @@ class Course: sdate:str enddate:str sem:str - crn:int + crn:int code:int section:int short:str long:str frequency:str - desc:str + desc:str raw:str pre:list co:list major:str school:str lec:str - #Info will be an array of strings: - # [crn, major, code, section, credits, name, days, stime, etime, max, curr, rem, profs, sdate, enddate, loc] + #Info will be an array of strings: + # [crn, major, code, section, credits, name, days, stime, + # etime, max, curr, rem, profs, sdate, enddate, loc] def __init__(self, info): self.crn = info[0] @@ -50,7 +49,7 @@ def __init__(self, info): self.sdate = info[13] self.enddate = info[14] self.loc = info[15] - self.long = self.processName(self.name) + self.long = self.process_name(self.name) self.frequency = "" self.short = self.major + '-' + self.code self.lec = "LEC" @@ -60,63 +59,72 @@ def __init__(self, info): self.co = list() self.school = "" self.sem = "" - - def processName(self, name:str) -> str: + + def process_name(self, name:str) -> str: tmp = name.split() for i in range(0, len(tmp), 1): if not tmp[i].isalpha(): - continue + continue tmp[i]= tmp[i][:1].upper() + tmp[i][1:].lower() return ' '.join(tmp) - def addSemester(self, semester): + + def add_semester(self, semester): self.sem = semester.upper() - def addReqs(self, pre:list=[], co:list=[], raw:str="", desc: str=""): + + def add_reqs(self, pre:list=None, co:list=None, raw:str="", desc: str=""): + if pre is None: + pre = [] + if co is None: + co = [] self.desc = desc self.raw = raw self.pre = copy.deepcopy(pre) self.co = copy.deepcopy(co) - - def addReqsFromList(self, info: list=[]): + + def add_reqs_from_list(self, info:list=None): + if info is None: + info = [] self.pre = info[0] self.co = info[1] self.raw = info[2] self.desc = info[3] + def print(self): for attr, value in self.__dict__.items(): print(attr, " : ", value) - #Turn the class back into a list. + #Turn the class back into a list. #Because of the diffs in how we store vs how we want it to be, need to do a lot of swapping #Maybe there's a diff way than doing this, hopefully there is def decompose(self) -> list[str]: - retList = [] - retList.append(self.name) - retList.append(self.lec) - retList.append(self.credits) - retList.append(self.days) - retList.append(self.stime) - retList.append(self.etime) - retList.append(self.profs) - retList.append(self.loc) - retList.append(self.max) - retList.append(self.curr) - retList.append(self.rem) - retList.append(self.major) - retList.append(self.sdate) - retList.append(self.enddate) - retList.append(self.sem) - retList.append(self.crn) - retList.append(self.code) - retList.append(self.section) - retList.append(self.short) - retList.append(self.long) - retList.append(self.desc) - retList.append(self.raw) - retList.append(self.frequency) - retList.append(self.pre) - retList.append(self.co) - retList.append(self.school) - return retList - + ret_list = [] + ret_list.append(self.name) + ret_list.append(self.lec) + ret_list.append(self.credits) + ret_list.append(self.days) + ret_list.append(self.stime) + ret_list.append(self.etime) + ret_list.append(self.profs) + ret_list.append(self.loc) + ret_list.append(self.max) + ret_list.append(self.curr) + ret_list.append(self.rem) + ret_list.append(self.major) + ret_list.append(self.sdate) + ret_list.append(self.enddate) + ret_list.append(self.sem) + ret_list.append(self.crn) + ret_list.append(self.code) + ret_list.append(self.section) + ret_list.append(self.short) + ret_list.append(self.long) + ret_list.append(self.desc) + ret_list.append(self.raw) + ret_list.append(self.frequency) + ret_list.append(self.pre) + ret_list.append(self.co) + ret_list.append(self.school) + return ret_list + def list_to_class(self, row): self.name = row[0] self.lec = row[1] @@ -144,7 +152,7 @@ def list_to_class(self, row): self.co = row[24] self.school = row[25] - def addSchool(self, school): + def add_school(self, school): self.school = school def __lt__(self, other): #Note that we will maybe need to compare times? Idk how to handle the case where the classes @@ -155,6 +163,6 @@ def __lt__(self, other): if self.code > other.code: return self.code > other.code return self.section > other.section - + def __str__(self): - return self.name \ No newline at end of file + return self.name diff --git a/rpi_data/modules/csv_to_course.py b/rpi_data/modules/csv_to_course.py index 9b0594d91..c70fb6787 100644 --- a/rpi_data/modules/csv_to_course.py +++ b/rpi_data/modules/csv_to_course.py @@ -1,10 +1,10 @@ +import os import csv from course import Course -import os -import pdb __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) -# This file takes our csv formatting and turns it into a course class type. If something goes wrong it's because you changed one of those two. +# This file takes our csv formatting and turns it into a course class type. +# If something goes wrong it's because you changed one of those two. def parse_csv(filename): courses = list() @@ -43,4 +43,4 @@ def parse_csv(filename): temp.co = row[24] temp.school = row[25] courses.append(temp) - return courses \ No newline at end of file + return courses diff --git a/rpi_data/modules/fetch_catalog_course_info.py b/rpi_data/modules/fetch_catalog_course_info.py index 250b44c6a..c91d2a425 100644 --- a/rpi_data/modules/fetch_catalog_course_info.py +++ b/rpi_data/modules/fetch_catalog_course_info.py @@ -1,13 +1,11 @@ -import requests as req import threading #https://docs.python.org/3/library/threading.html import unicodedata -import re -import regex #https://www.dataquest.io/blog/regex-cheatsheet/ -import json -from datetime import date -from time import time +import json from threading import Lock from io import StringIO +import re +import regex #https://www.dataquest.io/blog/regex-cheatsheet/ +import requests as req from bs4 import BeautifulSoup, SoupStrainer #https://www.crummy.com/software/BeautifulSoup/bs4/doc/ from lxml import etree @@ -34,11 +32,14 @@ "department": True, "level": True, "full_name": True, - "short_name": True, # custom, requires department and level to be true. Use this to join with SIS data. + # custom, requires department and level to be true. Use this to join with SIS data. + "short_name": True, "description": True, "prerequisites": True, # custom "corequisites": True, # custom - "raw_precoreqs": True, # If either prereq or coreq is true, then this must be true cause the client needs to look at this field to understand the other two + # If either prereq or coreq is true, then this must be true cause the client needs + # to look at this field to understand the other two + "raw_precoreqs": True, "offer_frequency": True, "cross_listed": False, "graded": False, @@ -51,7 +52,8 @@ prolog_and_root_ele_regex = re.compile("^(?P<\?xml.*?\?>)?\s*(?P)") # group the most specific regex patterns first, then the more general ones for last -# goal is to capture classes that are loosely of the form "Prerequisites: [CAPTURE COURSE LISTINGS TEXT]", +# goal is to capture classes that are loosely of the form +# "Prerequisites: [CAPTURE COURSE LISTINGS TEXT]", # but does not capture classes explicitly stated to be corequisites. Tries to remove # periods, trailing and leading space. explicit_prereqs_include_syntax_regex = "(?:^\s*Prerequisites? include:?\s?(.*))" diff --git a/rpi_data/modules/headless_login.py b/rpi_data/modules/headless_login.py index 715d19d4f..a49cb6981 100644 --- a/rpi_data/modules/headless_login.py +++ b/rpi_data/modules/headless_login.py @@ -1,3 +1,6 @@ +import time +import os +import sys import selenium as sel from selenium import webdriver from selenium.webdriver.common.by import By @@ -5,30 +8,39 @@ from selenium.webdriver.firefox.options import Options from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC -import time -import os -import sys # Remember to add enviromental variables named rcsid and rcspw with your account info!!! # # THINGS THAT CAN POTENTIALLY GO WRONG HERE AND HOW TO FIX THEM: -# -# - If the RPI login website changes at all, it's very likely that the login will break. Fixing might involve changing what element selenium looks for. -# - DUO likes to change things. If they implement another 2FA type or add extra buttons for some reason you'll have to add more checks and button presses -# - Selenium errors can occur if your internet is slow or if you have multiple browser instances open, so try to avoid this -# -# - You need to install firefox (I hate Google Chrome, and you should too). If you change it to be a Chrome instance instead it probably won't work from my experience -# - To fix these things you can comment this line: "options.add_argument("--headless")" in the parse_runner file to see what goes wrong if python doesn't throw anything -# - Try restarting python/vscode or even your computer if it's throwing something really weird for no reason +# +# - If the RPI login website changes at all, it's very likely that the login will break. +# Fixing might involve changing what element selenium looks for. +# - DUO likes to change things. If they implement another 2FA type or add extra buttons +# for some reason you'll have to add more checks and button presses +# - Selenium errors can occur if your internet is slow or if you have multiple browser +# instances open, so try to avoid this +# +# - You need to install firefox (I hate Google Chrome, and you should too). If you change +# it to be a Chrome instance instead it probably won't work from my experience +# - To fix these things you can comment this line: "options.add_argument("--headless")" +# in the parse_runner file to see what goes wrong if python doesn't throw anything +# - Try restarting python/vscode or even your computer if it's throwing something really +# weird for no reason # - You can try sending me a message on discord @gcm as a last resort def login(driver): - URL = "http://sis.rpi.edu" - driver.get(URL) # uses a selenium webdriver to go to the sis website, which then redirects to the rcs auth website - username_box = driver.find_element(by=By.NAME, value = "j_username") # creates a variable which contains an element type, so that we can interact with it, j_username is the username text box - password_box = driver.find_element(by=By.NAME, value = "j_password") # j_password is the password box - submit = driver.find_element(by=By.NAME, value = "_eventId_proceed") # _eventId_proceed is the submit button + url = "http://sis.rpi.edu" + # uses a selenium webdriver to go to the sis website, + # which then redirects to the rcs auth website + driver.get(url) + # creates a variable which contains an element type, so that we can + # interact with it, j_username is the username text box + username_box = driver.find_element(by=By.NAME, value = "j_username") + # j_password is the password box + password_box = driver.find_element(by=By.NAME, value = "j_password") + # _eventId_proceed is the submit button + submit = driver.find_element(by=By.NAME, value = "_eventId_proceed") username = os.environ.get("rcsid", "NONEFOUND") password = os.environ.get("rcspw", "NONEFOUND") if (username == "NONEFOUND" or password == "NONEFOUND"): @@ -38,14 +50,18 @@ def login(driver): username_box.send_keys(username) # enters the username password_box.send_keys(password) # enters the password submit.click() # click the submit button - while ("duosecurity" not in driver.current_url): # if you entered details incorrectly, the loop will be entered as you aren't on the duo verfication website (redo what we did before) + # if you entered details incorrectly, the loop will be entered + # as you aren't on the duo verfication website (redo what we did before) + while "duosecurity" not in driver.current_url: print("User or Password Incorrect.") - username_box = driver.find_element(by=By.NAME, value = "j_username") # we have to redefine the variables because the webpage reloads + # we have to redefine the variables because the webpage reloads + username_box = driver.find_element(by=By.NAME, value = "j_username") password_box = driver.find_element(by=By.NAME, value = "j_password") submit = driver.find_element(by=By.NAME, value = "_eventId_proceed") username = input("Enter Username: ") password = input("Enter Password: ") - username_box.clear() # the username box by default has your previous username entered, so we clear it + # the username box by default has your previous username entered, so we clear it + username_box.clear() username_box.send_keys(username) password_box.send_keys(password) submit.click() @@ -53,20 +69,24 @@ def login(driver): time.sleep(.1) options = driver.find_element(By.XPATH, '/html/body/div/div/div[1]/div/div[2]/div[7]/a') options.click() - while len(driver.find_elements(By.XPATH, '/html/body/div/div/div[1]/div/div[1]/ul/li[1]/a')) == 0: + while len(driver.find_elements(By.XPATH, '/html/body/div/div/div[1]/div/div[1]/ul/li[1]/a'))==0: time.sleep(.1) duo_option = driver.find_element(By.XPATH, '/html/body/div/div/div[1]/div/div[1]/ul/li[1]/a') duo_option.click() while len(driver.find_elements(By.XPATH, '/html/body/div/div/div[1]/div/div[2]/div[3]')) == 0: time.sleep(.1) - print("Your DUO code: "+ driver.find_element(by= By.XPATH, value = "/html/body/div/div/div[1]/div/div[2]/div[3]").text) # print the duo code - while len(driver.find_elements(By.XPATH, '//*[@id="trust-browser-button"]'))==0: # we need to press the trust browser button, so we wait until that shows up + print("Your DUO code: " + + driver.find_element(by= By.XPATH, + value="/html/body/div/div/div[1]/div/div[2]/div[3]").text) # print the duo code + # we need to press the trust browser button, so we wait until that shows up + while len(driver.find_elements(By.XPATH, '//*[@id="trust-browser-button"]'))==0: time.sleep(.1) - trust_button = driver.find_element(By.XPATH, '//*[@id="trust-browser-button"]') #find and click it + #find and click it + trust_button = driver.find_element(By.XPATH, '//*[@id="trust-browser-button"]') trust_button.click() time.sleep(3) - if (driver.current_url == "https://sis.rpi.edu/rss/twbkwbis.P_GenMenu?name=bmenu.P_MainMnu"): # check if we're in the right place + # check if we're in the right place + if driver.current_url == "https://sis.rpi.edu/rss/twbkwbis.P_GenMenu?name=bmenu.P_MainMnu": return "Success" - else: - print("login failed") - return "Failure" + print("login failed") + return "Failure" diff --git a/rpi_data/modules/new_parse.py b/rpi_data/modules/new_parse.py index ff3c7aa95..b8193614a 100755 --- a/rpi_data/modules/new_parse.py +++ b/rpi_data/modules/new_parse.py @@ -1,4 +1,11 @@ #!/usr/bin/env python +import time +import copy +from concurrent.futures import ThreadPoolExecutor +import sys +from bs4 import BeautifulSoup as bs +import pandas as pd +import pdb import requests import selenium as sel from selenium import webdriver @@ -6,35 +13,34 @@ from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.chrome.options import Options from selenium.webdriver.support.ui import Select -import time -from bs4 import BeautifulSoup as bs -import pandas as pd -import pdb -import copy from course import Course -from concurrent.futures import ThreadPoolExecutor -import sys import headless_login as login -# from lxml, based on the code from the quacs scraper and the other scraper, we will prob need to parse xml markup -# term format: spring2023, fall2023, summer2023, hartford2023, enrichment2023 -# TROUBLESHOOTING: remove the line "options.add_argument("--headless")" to see where the script might be stalling -# TROUBLESHOOTING: If DUO changes their website the parser will break (but in an easy to fix way, like by adding an extra button click) SIS hasn't been substantially changed since 2006 so it probably won't be changed any time soon. -# The sisCourseSearch function serves a similar purpose as the courseUpdate function, so if you make any changes to the the top half of either (before the for loop), try to do it for both +#from lxml, based on the code from the quacs scraper and the other scraper, +# we will prob need to parse xml markup +#term format: spring2023, fall2023, summer2023, hartford2023, enrichment2023 +#TROUBLESHOOTING: remove the line "options.add_argument("--headless")" +# to see where the script might be stalling +#TROUBLESHOOTING: If DUO changes their website the parser will break +# (but in an easy to fix way, like by adding an extra button click) SIS +# hasn't been substantially changed since 2006 so it probably won't be changed any time soon. +#The sisCourseSearch function serves a similar purpose as the courseUpdate +# function, so if you make any changes to the the top half of either (before the for loop), +# try to do it for both -def genBasevalue(term): #this function returns the code sis uses for a specific term +def gen_base_value(term): #this function returns the code sis uses for a specific term basevalue = 200000 #this number will represent the term we are looking at while True: #this will add the term code to the last digit, making sure that the term exists try: # add the month value - if ("spring" in term): + if "spring" in term: basevalue += 1 - elif ("fall" in term): - basevalue += 9 - elif ("summer" in term): + elif "fall" in term: + basevalue += 9 + elif "summer" in term: basevalue += 5 - elif ("hartford" in term): + elif "hartford" in term: basevalue += 10 - elif ("enrichment" in term): + elif "enrichment" in term: basevalue += 12 else: raise Exception("term not found") @@ -42,24 +48,31 @@ def genBasevalue(term): #this function returns the code sis uses for a specific term = input("Your term may be incorrect, enter the correct term here:") else: break - year = int(term[-2])*10 + int(term[-1]) # this is the last two digits of the year + year = int(term[-2])*10 + int(term[-1]) # this is the last two digits of the year basevalue += year * 100 #this makes the basevalue show our year return basevalue -def sisCourseSearch(driver, term): #main loop of the parser, goes to the course search, selects the desired term, and then loops through each subject to grab the course tables - info = list() - course_codes_dict = findAllSubjectCodes(driver) +def sis_course_search(driver, term): + """ + main loop of the parser, goes to the course search, selects the desired term, + and then loops through each subject to grab the course tables + """ + info = [] + course_codes_dict = find_all_subject_codes(driver) url = "https://sis.rpi.edu/rss/bwskfcls.p_sel_crse_search" driver.get(url) - select = Select(driver.find_element(by=By.ID, value = "term_input_id")) # term selection dropdown - basevalue = genBasevalue(term) + # term selection dropdown + select = Select(driver.find_element(by=By.ID, value = "term_input_id")) + basevalue = gen_base_value(term) try: select.select_by_value(str(basevalue)) # select term based on the basevalue except Exception: print("The term you entered does not exist.") sys.exit() - driver.find_element(by = By.XPATH, value = "/html/body/div[3]/form/input[2]").click() # submit term button - subject_select = Select(driver.find_element(by=By.XPATH, value = '//*[@id="subj_id"]')) # select subject dropdown + # submit term button + driver.find_element(by = By.XPATH, value = "/html/body/div[3]/form/input[2]").click() + # select subject dropdown + subject_select = Select(driver.find_element(by=By.XPATH, value = '//*[@id="subj_id"]')) subjects = subject_select.options subject = "" key = str(basevalue)[:4] + "-" @@ -68,10 +81,11 @@ def sisCourseSearch(driver, term): #main loop of the parser, goes to the course subject_select.select_by_index(i) # selects a subject driver.find_element(by = By.NAME, value = 'SUB_BTN').click() # submits course search print("Getting course info") - courses = getCourseInfo(driver, key, course_codes_dict) # creates a list of course objects + courses = get_course_info(driver, key, course_codes_dict) # creates a list of course objects with ThreadPoolExecutor(max_workers=50) as pool: - pool.map(getReqForClass, courses) - [info.append(i) for i in courses] # appends each course to our final list + pool.map(get_req_for_class, courses) + for i in courses: + info.append(i) # appends each course to our final list subject = info[len(info)-1].major # gets the subject we just parsed driver.get(url) # goes back to the start end = time.time() @@ -82,24 +96,26 @@ def sisCourseSearch(driver, term): #main loop of the parser, goes to the course driver.find_element(by = By.XPATH, value = "/html/body/div[3]/form/input[2]").click() subject_select = Select(driver.find_element(by=By.XPATH, value = '//*[@id="subj_id"]')) info.sort() - #Because we end up sorting in reverse order, we need to reverse the list to get the correct order + #Because we end up sorting in reverse order, reverse the list to get the correct order info.reverse() return info -def findAllSubjectCodes(driver) -> dict(): - url = 'https://catalog.rpi.edu/content.php?catoid=26&navoid=670&hl=%22subject%22&returnto=search' #link to a list of schools with their subject codes +def find_all_subject_codes(driver) -> dict(): + #link to a list of schools with their subject codes + url='https://catalog.rpi.edu/content.php?catoid=26&navoid=670&hl=%22subject%22&returnto=search' driver.get(url) - code_school_dict = dict() # We store in a dictionary that has subject code as the key and school as the value + # We store in a dictionary that has subject code as the key and school as the value + code_school_dict = dict() html = driver.page_source soup = bs(html, 'html.parser') ptag = soup.find_all('p') # Entire text of page basically look_at = [] - for all in ptag: # finds all things that are important - if all.find('strong'): - look_at.append(all) - for all in look_at: # in every important part + for all_things in ptag: # finds all things that are important + if all_things.find('strong'): + look_at.append(all_things) + for all_things in look_at: # in every important part school = "" - for tags in all: # look at each school + for tags in all_things: # look at each school if tags.name == "strong": # if bold, it's the school name school = tags.text school_first = school.split(' ') @@ -116,19 +132,23 @@ def findAllSubjectCodes(driver) -> dict(): if "\xa0" in line: line = line.replace("\xa0", ' ') info = line.split(' ') - code_school_dict[info[0]] = school # uses the current value of school for the dictionary (kind of backwards, but better for our use case) + # uses the current value of school for the dictionary + # (kind of backwards, but better for our use case) + code_school_dict[info[0]] = school return code_school_dict - -#For courses that do not have a crn. 99% of the time, these will be lab blocks, test blocks, or recitations -def processSpecial(info, prevrow) -> list[str]: +def process_special(info, prevrow) -> list[str]: + """ + For courses that do not have a crn, 99% of the time, these + will be lab blocks, test blocks, or recitations + """ #If this is ever called on an incorrect course. - #Shouldn't happen but who knows - if prevrow == None: + #Shouldn't happen but who knows + if prevrow is None: print("course has no crn but first in major") return info #Maybe just exit the program instead? - tmp = formatTimes(info) - tmp[18] = formatTeachers(tmp[18]) + tmp = format_times(info) + tmp[18] = format_teachers(tmp[18]) info = prevrow info[6] = tmp[6] info[7] = tmp[7] @@ -136,8 +156,12 @@ def processSpecial(info, prevrow) -> list[str]: info[12] = tmp[18] info[15] = tmp[20] return info -#Given a string contaings the profs for a class, return a string containing only the last names of the profs seperated by a slash. -def formatTeachers(profs : str) -> str: + +def format_teachers(profs : str) -> str: + """ + Given a string contaings the profs for a class, return a string + containing only the last names of the profs seperated by a slash. + """ if profs == "TBA": return profs index = profs.find("(P)") @@ -152,9 +176,14 @@ def formatTeachers(profs : str) -> str: profs += tmp[i] + "/" profs = profs[:len(profs)-1] return profs -#Format the times of the classes into a start and ending time. ie 2:00pm-3:50pm becomes 2:00pm as the start time and 3:50pm as the end time -def formatTimes(info : list[str]) -> list[str]: - #Special case where the time is tba or there isn't a valid time see admin 1030 or admin 1100 in spring 2024 + +def format_times(info : list[str]) -> list[str]: + """ + Format the times of the classes into a start and ending time. + ie 2:00pm-3:50pm becomes 2:00pm as the start time and 3:50pm as the end time + """ + #Special case where the time is tba or there isn't a valid + #time see admin 1030 or admin 1100 in spring 2024 if(info[7] == "TBA" or ':' not in info[7]): info.insert(8, "") info[7] = "" @@ -163,92 +192,119 @@ def formatTimes(info : list[str]) -> list[str]: if info[7] != "": while " " in info[7]: info[7] = info[7].replace(" ", "") - splitTime = info[7].split('-') - start = splitTime[0].upper() - end = splitTime[1].upper() - if (start[0] == '0'): + split_time = info[7].split('-') + start = split_time[0].upper() + end = split_time[1].upper() + if start[0] == '0': start = start[1:] - if (end[0] == '0'): + if end[0] == '0': end = end[1:] info.insert(7,start) info.insert(8,end) #Pop to remove original time info.pop(9) return info -#Given the starting and ending date of a class, ie 01/08-04/24, turn it into a format the backend likes - 2024-01-08, 2024-04-24 -def formatDate(info : list, year : str): - splitDate = info[13].split('-') - sdate = splitDate[0].split('/') + +def format_date(info : list, year : str): + """ + Given the starting and ending date of a class, ie 01/08-04/24, + turn it into a format the backend likes - 2024-01-08, 2024-04-24 + """ + split_date = info[13].split('-') + sdate = split_date[0].split('/') sdate = '-'.join(sdate) - enddate = splitDate[1].split('/') + enddate = split_date[1].split('/') enddate = '-'.join(enddate) #Sdate and enddate are the month and days of a semester, eg 01-08 and 05-04 info.insert(13, year + sdate) info.insert(14, year + enddate) info.pop(15) -#Turn the credits into an int, pick the greatest credit value if there is a range, eg 0.000-16.000 is returned as 16 -def formatCredits(info): + +def format_credits(info): + """ + Turn the credits into an int, pick the greatest credit value + if there is a range, eg 0.000-16.000 is returned as 16 + """ if '-' in info[4]: return int(float(info[4].split('-')[1])) return int(float(info[4])) -#Given a row in sis, process the data in said row including crn, course code, days, seats, etc -#Note that we remove a lot of info from the row in SIS, this is to match the format that the csv is expecting -#See Fall 2023 or any other csv in the repository. -def processRow(data: list[str], prevrow: list[str], year: int) -> list[str]: +def process_row(data: list[str], prevrow: list[str], year: int) -> list[str]: + """ + Given a row in sis, process the data in said row including crn, course code, days, seats, etc + Note that we remove a lot of info from the row in SIS, this is to match the format that the + csv is expecting, see Fall 2023 or any other csv in the repository. + """ info = [] - #The first and last elements of the row are useful to us as other parts of the application handle those + #The first and last elements of the row are useful to us as + #other parts of the application handle those for i in range(1, len(data) - 1, 1): #Edge case where the registrar decides to make a column an inconcsistent width. #See MGMT 2940 - Readings in MGMT in spring 2024. - #TODO: Accomodate for colspans different than 2. + #TODO: Accomodate for colspans different than 2. #See https://stackoverflow.com/questions/13263373/beautifulsoup-parsing-tag-table-html-especially-colspan-and-rowspan to start - if(data[i].has_attr("colspan")): + if data[i].has_attr("colspan"): + info.append("TBA") info.append("TBA") - info.append("TBA") else: info.append(data[i].text) if len(info) != 21: print("Length error in: ") print(info[0]) - # info[0] is crn, info[1] is major, 2 - course code, 3- section, 4 - if class is on campus (most are), 5 - credits, 6 - class name - #info[7] is days, info[8] is time, info[9] is total course capactiy, info[10] is number of students enrolled, - #info[11] is the number of seats left, info[12] - waitlist capacity, info[13] - waitlist enrolled, info[14] - waitlist spots left, - #info[15] - crosslist capacity, info[16] - crosslist enrolled, info[17] - crosslist seats left, - #info[18] are the profs, info[19] are days of the sem that the course spans, and info[20] is location - #Remove index[4] because most classes are on campus, with exceptions for some grad and doctoral courses. + #info[0] is crn, info[1] is major, 2 - course code, 3- section, + # 4 - if class is on campus (most are), 5 - credits, 6 - class name + #info[7] is days, info[8] is time, info[9] is total course capactiy, + #info[10] is number of students enrolled, + #info[11] is the number of seats left, info[12] - waitlist capacity, + #info[13] - waitlist enrolled, info[14] - waitlist spots left, + #info[15] - crosslist capacity, info[16] - crosslist enrolled, + #info[17] - crosslist seats left, + #info[18] are the profs, info[19] are days of the sem that the + #course spans, and info[20] is location + #Remove index[4] because most classes are on campus, with + #exceptions for some grad and doctoral courses. info.pop(4) - + #Note that this will shift the above info down by 1 to - # info[0] crn, info[1] major, 2 - course code, 3- section, 4 - credits, 5 - class name - #info[6] is days, info[7] time, info[8] - info[16] actual, waitlist, and crosslist capacity, enrolled, remaining + #info[0] crn, info[1] major, 2 - course code, 3- section, 4 - credits, 5 - class name + #info[6] is days, info[7] time, info[8] - info[16] actual, + #waitlist, and crosslist capacity, enrolled, remaining #info[17] profs, info[18] days of sem, and info[19] location #The above info is what we are working with for the rest of the method - #If the crn is empty, then the course is most likely a lab, test, or recitation, so process this row seperately. + #If the crn is empty, then the course is most likely a lab, test, + # or recitation, so process this row seperately. if str(info[0]) == '\xa0': - info = processSpecial(info, prevrow) + info = process_special(info, prevrow) return info #Some admin and grad courses won't have days of the week #Also the backend doesn't like the days of the week being TBA if (info[6] == '\xa0' or info[6] == "TBA"): info[6] = "" - #Generally speaking methods that affect info should come in the order that the affect elements, ie - #time formatitng should come before prof or date formatting because time is at info[6] while date and - #prof is after. Not doing this can lead to the scraper crashing on some edge cases (see admin 1030 in spring 2024) - formatTimes(info) + #Generally speaking methods that affect info should come in + #the order that the affect elements, ie + #time formatitng should come before prof or date + #formatting because time is at info[6] while date and + #prof is after. Not doing this can lead to the scraper + #crashing on some edge cases (see admin 1030 in spring 2024) + format_times(info) #Remove waitlist and crosslist stuff info = info[:12] + info[18:] #Split the date into start and end date - formatDate(info, year) - info[12] = formatTeachers(info[12]) + format_date(info, year) + info[12] = format_teachers(info[12]) #Some classes have a credit value ranging from 0-12, just pick the biggest credit value - #We do this instead of keeping the range because the backend does not like having a string for the credit value. - info[4] = formatCredits(info) + #We do this instead of keeping the range because the backend does not like having a + #string for the credit value. + info[4] = format_credits(info) return info -#Given a course, return the semester followed by the year, eg if the start date of a course is 2024-01-08, then this will return SPRING 2024 -def getStrSemester(c : Course) -> str: - val = str(getSemester(c)) + +def get_str_semester(c : Course) -> str: + """ + Given a course, return the semester followed by the year, eg if the start date of a + course is 2024-01-08, then this will return SPRING 2024 + """ + val = str(get_semester(c)) date = val[:4] if val[4:] == "01": date = "SPRING " + date @@ -261,8 +317,13 @@ def getStrSemester(c : Course) -> str: elif val[4:] == "10": date = "HARTFORD " + date return date -#Given a school year and a dictionary of every major to names of their schools, parse all of the info for every course in a major. -def getCourseInfo(driver, year:str, schools : dict) -> list: + + +def get_course_info(driver, year:str, schools : dict) -> list: + """ + Given a school year and a dictionary of every major to names of + their schools, parse all of the info for every course in a major. + """ html = driver.page_source soup = bs(html, 'html.parser') table = soup.find('table', class_='datadisplaytable') @@ -272,89 +333,109 @@ def getCourseInfo(driver, year:str, schools : dict) -> list: for row in rows: data = row.find_all("td") if len(data) != 0: - tmpCourse = (processRow(data, prevrow, year)) - prevrow = copy.deepcopy(tmpCourse) - c = Course(tmpCourse) - c.addSemester(getStrSemester(c)) + tmp_course = process_row(data, prevrow, year) + prevrow = copy.deepcopy(tmp_course) + c = Course(tmp_course) + c.add_semester(get_str_semester(c)) if c.major in schools: - c.addSchool(schools[c.major]) + c.add_school(schools[c.major]) else: - #A catch all for courses that aren't listed on the offical catalog, such as ADMN or BSUN courses. - #If in the future there are too many of these were it shouldn't be, then we will have to find a better solution - c.addSchool("Interdisciplinary and Other") + #A catch all for courses that aren't listed on the + #offical catalog, such as ADMN or BSUN courses. + #If in the future there are too many of these were + #it shouldn't be, then we will have to find a better solution + c.add_school("Interdisciplinary and Other") courses.append(c) return courses -#Given a url for a course, as well as the course code and major, return a list of prereqs, coreqs, and description of the course -#Eg. ITWS 2110 - https://sis.rpi.edu/rss/bwckctlg.p_disp_course_detail?cat_term_in=202401&subj_code_in=ITWS&crse_numb_in=2110 -# Prereqs - ITWS 1100 -# Coreqs - CSCI 1200 -# Raw - Undergraduate level CSCI 1200 Minimum Grade of D and Undergraduate level ITWS 1100 Minimum Grade of D or Prerequisite Override 100 -# Description - This course involves a study of the methods used to extract and deliver dynamic information on the World Wide Web. -# The course uses a hands-on approach in which students actively develop Web-based software systems. -# Additional topics include installation, configuration, and management of Web servers. -# Students are required to have access to a PC on which they can install software such as a Web server and various programming environments. -def getReqFromLink(webres, courseCode, major) -> list: + +def get_req_from_link(webres, course_code, major) -> list: + """ + Given a url for a course, as well as the course code and major, + return a list of prereqs, coreqs, and description of the course + Eg. ITWS 2110 - https://sis.rpi.edu/rss/bwckctlg.p_disp_course_detail?cat_term_in=202401&subj_code_in=ITWS&crse_numb_in=2110 + Prereqs - ITWS 1100 + Coreqs - CSCI 1200 + Raw - Undergraduate level CSCI 1200 Minimum Grade of D and + Undergraduate level ITWS 1100 Minimum Grade of D or Prerequisite Override 100 + Description - This course involves a study of the methods used to extract and + deliver dynamic information on the World Wide Web. + The course uses a hands-on approach in which students actively develop Web-based software + systems. Additional topics include installation, configuration, and management of Web servers. + Students are required to have access to a PC on which they can install software such + as a Web server and various programming environments. + """ page = webres.content soup = bs(page, "html.parser") body = soup.find('td', class_='ntdefault') #The page is full of \n\n's for some reason, and this nicely splits it into sections - classInfo = body.text.strip('\n\n').split('\n\n') - for i in range(0,len(classInfo),1): - while '\n' in classInfo[i]: + class_info = body.text.strip('\n\n').split('\n\n') + for i in range(0,len(class_info),1): + while '\n' in class_info[i]: #Some \n's can make it into the parsed data, so we need to get rid of them. - classInfo[i] = classInfo[i].replace('\n','') + class_info[i] = class_info[i].replace('\n','') key = "Prerequisites/Corequisites" - preKey = "Prerequisite" + pre_key = "Prerequisite" prereqs = "" coreqs = "" raw = "" - desc = classInfo[0] - #If the course does not have a description, usually this menas that classInfo[0] will be the credit value. + desc = class_info[0] + #If the course does not have a description, usually this means that + #classInfo[0] will be the credit value. if desc.strip()[0].isdigit(): desc = "" - for i in range(1, len(classInfo)): - if key in classInfo[i].strip(): - combo = classInfo[i].strip() + for i in range(1, len(class_info)): + if key in class_info[i].strip(): + combo = class_info[i].strip() combo = combo[len(key):] - coKey = "Corequisite" - if coKey in combo and preKey in combo: - coreqs = combo[combo.find(coKey) + len(coKey):] - prereqs = combo[len(preKey): combo.find(coKey)] - elif coKey in combo: - coreqs = combo[combo.find(coKey) + len(coKey):] - elif preKey in combo: - prereqs = combo[len(preKey):] + co_key = "Corequisite" + if co_key in combo and pre_key in combo: + coreqs = combo[combo.find(co_key) + len(co_key):] + prereqs = combo[len(pre_key): combo.find(co_key)] + elif co_key in combo: + coreqs = combo[combo.find(co_key) + len(co_key):] + elif pre_key in combo: + prereqs = combo[len(pre_key):] else: #Default case where someone forgets the words we're looking for - #Note that there are still more edge cases(looking at you csci 6560 and 2110 in spring 2024) + #Note that there are still more edge cases + #(looking at you csci 6560 and 2110 in spring 2024) prereqs = combo prereqs = prereqs[prereqs.find(' '):255].strip() coreqs = coreqs[coreqs.find(' '):255].strip() - if classInfo[i].strip() == (preKey + "s:"): - raw = classInfo[i+1].strip() - retList = [prereqs, coreqs, raw, desc] - return retList -#Add the prereqs for a course to that course -def getReqForClass(course: Course) -> None: - semester = getSemester(course) - url = "https://sis.rpi.edu/rss/bwckctlg.p_disp_course_detail?cat_term_in={}&subj_code_in={}&crse_numb_in={}".format(semester, course.major, course.code) + if class_info[i].strip() == (pre_key + "s:"): + raw = class_info[i+1].strip() + ret_list = [prereqs, coreqs, raw, desc] + return ret_list + +def get_req_for_class(course: Course) -> None: + """ + Add the prereqs for a course to that course + """ + semester = get_semester(course) + url = f"https://sis.rpi.edu/rss/bwckctlg.p_disp_course_detail?cat_term_in={semester}&subj_code_in={course.major}&crse_numb_in=course.code{course.code}" session = requests.session() webres = session.get(url) - course.addReqsFromList(getReqFromLink(webres, course.code, course.major)) -#Given a course, return the basevalue of that course, eg 2024-01 is returned as 202401 -def getSemester(course: Course) -> int: + course.add_reqs_from_list(get_req_from_link(webres, course.code, course.major)) + +def get_semester(course: Course) -> int: + """ + Given a course, return the basevalue of that course, eg 2024-01 is returned as 202401 + """ dates = course.sdate.split("-") month = dates[1] year = dates[0] sem = year - if month == "08" or month == "09": + if month in set("08", "09"): sem += "09" else: sem += month return sem -#Given a list of courses, write the courses to a csv specified in the filename -def writeCSV(info:list, filename: str): - columnNames = ['course_name', 'course_type', 'course_credit_hours', + +def write_csv(info:list, filename: str): + """ + Given a list of courses, write the courses to a csv specified in the filename + """ + column_names = ['course_name', 'course_type', 'course_credit_hours', 'course_days_of_the_week', 'course_start_time', 'course_end_time', 'course_instructor', 'course_location', 'course_max_enroll', 'course_enrolled', 'course_remained', 'course_department', 'course_start_date', 'course_end_date', @@ -364,10 +445,11 @@ def writeCSV(info:list, filename: str): decomposed = [[]] * len(info) for i in range(0, len(info), 1): decomposed[i] = info[i].decompose() - df = pd.DataFrame(decomposed, columns = columnNames) + df = pd.DataFrame(decomposed, columns = column_names) df.to_csv(filename, index=False) -# This main function is helpful for running the full parser standalone, without needing environmental variables. +# This main function is helpful for running the full parser standalone, +# without needing environmental variables. def main(): options = Options() @@ -379,10 +461,9 @@ def main(): driver.implicitly_wait(2) login.login(driver) start = time.time() - final = sisCourseSearch(driver, "spring2024") + final = sis_course_search(driver, "spring2024") end = time.time() - writeCSV(final, "test.csv") + write_csv(final, "test.csv") print("Total Elapsed: " + str(end - start)) #main() - diff --git a/rpi_data/modules/parse_runner.py b/rpi_data/modules/parse_runner.py index 7bdd3ac29..aff95b68d 100644 --- a/rpi_data/modules/parse_runner.py +++ b/rpi_data/modules/parse_runner.py @@ -1,35 +1,38 @@ #!/usr/bin/env python +import os +import sys +from datetime import datetime +import time from selenium import webdriver from selenium.webdriver.firefox.options import Options import headless_login as login import new_parse as parser -import sys -from datetime import datetime import pytz from selenium.webdriver.support.ui import Select from selenium.webdriver.common.by import By from course import Course -import time import csv_to_course import selenium import pdb -import os # COMMON PONTS OF ERROR: -# +# # - If Login fails, check the headless_login file -# - If it fails in the middle of parsing that means there's some sis formatting BS going on (these problems are the hardest to fix because they don't make sense, -# check the getCourseInfo function and everything it calls, make sure to figure out what's going wrong) -# - If it fails in between subjects or on term selection that means they changed the website formatting, you'll have to check the courseUpdate function -# - remember to add a command line argument for the term that's being parsed, otherwise it defaults to Spring 2024 (the next term as of writing this comment) The formatting is "spring2024" +# - If it fails in the middle of parsing that means there's some sis formatting BS going on +# (these problems are the hardest to fix because they don't make sense, check the getCourseInfo +# function and everything it calls, make sure to figure out what's going wrong) +# - If it fails in between subjects or on term selection that means they changed the website +# formatting, you'll have to check the courseUpdate function +# - remember to add a command line argument for the term that's being parsed, otherwise +# it defaults to Spring 2024 (the next term as of writing this comment) The formatting is "spring2024" -def courseUpdate(driver, term, courses): - schools = parser.findAllSubjectCodes(driver) +def course_update(driver, term, courses): + schools = parser.find_all_subject_codes(driver) full_info = list() url = "https://sis.rpi.edu/rss/bwskfcls.p_sel_crse_search" driver.get(url) select = Select(driver.find_element(by=By.ID, value = "term_input_id")) - basevalue = parser.genBasevalue(term) + basevalue = parser.gen_base_value(term) select.select_by_value(str(basevalue)) driver.find_element(by = By.XPATH, value = "/html/body/div[3]/form/input[2]").click() subject_select = Select(driver.find_element(by=By.XPATH, value = '//*[@id="subj_id"]')) @@ -38,13 +41,14 @@ def courseUpdate(driver, term, courses): for i in range(len(subjects)): subject_select.select_by_index(i) driver.find_element(by = By.NAME, value = 'SUB_BTN').click() - info = parser.getCourseInfo(driver, key, schools) + info = parser.get_course_info(driver, key, schools) driver.get(url) select = Select(driver.find_element(by=By.ID, value = "term_input_id")) select.select_by_value(str(basevalue)) driver.find_element(by = By.XPATH, value = "/html/body/div[3]/form/input[2]").click() subject_select = Select(driver.find_element(by=By.XPATH, value = '//*[@id="subj_id"]')) - [full_info.append(i) for i in info] + for i in info: + full_info.append(i) full_info.sort() full_info.reverse() x = 0 @@ -60,30 +64,31 @@ def courseUpdate(driver, term, courses): for i in range(len(full_info)): temp_tuple = tuple([full_info[i].crn, full_info[i].stime]) full_check.append(temp_tuple) - full_info[i].addSemester(parser.getStrSemester(full_info[i])) - if (temp_tuple not in check_dict.keys()): + full_info[i].addSemester(parser.get_str_semester(full_info[i])) + if temp_tuple not in check_dict: check_dict[temp_tuple] = full_info[i] - print("Error: course " + check_dict[temp_tuple].name + " " + check_dict[temp_tuple].crn + " out of order, adding new course") - parser.getReqForClass(check_dict[temp_tuple]) + print("Error: course " + check_dict[temp_tuple].name + " " + + check_dict[temp_tuple].crn + " out of order, adding new course") + parser.get_req_for_class(check_dict[temp_tuple]) continue new_class = full_info[i].decompose() old_class = check_dict[temp_tuple].decompose() for i in range(len(new_class)): - if (i == 20): + if i == 20: break - if (i == 4 or i == 15): + if i in (4, 15): continue - if (old_class[i] != new_class[i]): + if old_class[i] != new_class[i]: old_class[i] = new_class[i] check_dict[temp_tuple].list_to_class(old_class) courses = list() for i in range(len(check_dict.keys())): - if (list(check_dict.keys())[i] in full_check): + if list(check_dict.keys())[i] in full_check: courses.append(check_dict[list(check_dict.keys())[i]]) else: print("removed CRN: "+ list(check_dict.keys())[i][0]) - + courses.sort() return courses @@ -92,7 +97,7 @@ def courseUpdate(driver, term, courses): if __name__ == "__main__": options = Options() options.add_argument("--headless") - if (len(sys.argv) == 1): + if len(sys.argv) == 1: print("Error: No command argument detected. Defaulting to Spring 2024") term = "spring2024" else: @@ -113,29 +118,27 @@ def courseUpdate(driver, term, courses): else: break print("Login Successful") - if (not os.path.isfile(endpath)): + if not os.path.isfile(endpath): print("Existing csv not found, doing full parse") - courses = parser.sisCourseSearch(driver, term) - parser.writeCSV(courses, endpath) + courses = parser.sis_course_search(driver, term) + parser.write_csv(courses, endpath) else: courses = csv_to_course.parse_csv(endpath) time_zone = pytz.timezone('America/New_York') i = 0 has_updated = False - while (True): - if (datetime.now(time_zone).strftime("%H") == "01"): + while True: + if datetime.now(time_zone).strftime("%H") == "01": has_updated = False - if (datetime.now(time_zone).strftime("%H") == "00" and not has_updated): + if datetime.now(time_zone).strftime("%H") == "00" and not has_updated: print("Doing midnight Update") - courses = parser.sisCourseSearch(driver, term) - parser.writeCSV(courses, endpath) + courses = parser.sis_course_search(driver, term) + parser.write_csv(courses, endpath) time.sleep(40) has_updated = True driver.get("http://sis.rpi.edu") - courses = courseUpdate(driver, term, courses) - parser.writeCSV(courses, endpath) + courses = course_update(driver, term, courses) + parser.write_csv(courses, endpath) i += 1 print("Update # " + str(i) + " Finished") time.sleep(60) - - diff --git a/src/api/api_models.py b/src/api/api_models.py index 49dfd2659..7f0a87ddc 100644 --- a/src/api/api_models.py +++ b/src/api/api_models.py @@ -1,5 +1,5 @@ -from pydantic import BaseModel from typing import Optional +from pydantic import BaseModel class SessionPydantic(BaseModel): @@ -14,7 +14,7 @@ class CourseDeletePydantic(BaseModel): cid: Optional[str] = None semester: str -class updateUser(BaseModel): +class UpdateUser(BaseModel): name:str sessionID:str email:str @@ -24,12 +24,12 @@ class updateUser(BaseModel): degree:str class UserPydantic(BaseModel): - name: str - email: str - phone: str - password: str - major: str - degree: str + name: str + email: str + phone: str + password: str + major: str + degree: str class UserDeletePydantic(BaseModel): sessionID: str @@ -45,6 +45,3 @@ class SubsemesterPydantic(BaseModel): class DefaultSemesterSetPydantic(BaseModel): default: str - - - diff --git a/src/api/app.py b/src/api/app.py index 31cf7fe4a..c80838ce6 100644 --- a/src/api/app.py +++ b/src/api/app.py @@ -1,14 +1,16 @@ #!/usr/bin/python3 -from fastapi import FastAPI, HTTPException, Request, Response, UploadFile, Form, File, Depends +import json +import os +from io import StringIO +from fastapi import FastAPI, Request, Response, UploadFile, Form, File, Depends from starlette.middleware.sessions import SessionMiddleware from fastapi_cache import FastAPICache from fastapi_cache.backends.inmemory import InMemoryBackend from fastapi_cache.decorator import cache from fastapi_cache.coder import PickleCoder -from fastapi import Depends from api_models import * -import db.connection as connection +from db import connection import db.classinfo as ClassInfo import db.courses as Courses import db.professor as All_professors @@ -17,21 +19,16 @@ import db.admin as AdminInfo import db.student_course_selection as CourseSelect import db.user as UserModel +import db.finals as Finals +import db.course_watchers as CourseWatch import controller.user as user_controller import controller.session as session_controller -import controller.userevent as event_controller -from io import StringIO -from sqlalchemy.orm import Session -import json -import os import pandas as pd from constants import Constants -""" -NOTE: on caching -on add of semester of change of data from GET -do a cache.clear() to ensure data integrity -""" +#NOTE: on caching +#on add of semester of change of data from GET +#do a cache.clear() to ensure data integrity app = FastAPI() app.add_middleware(SessionMiddleware, @@ -43,11 +40,13 @@ db_conn = connection.db class_info = ClassInfo.ClassInfo(db_conn) courses = Courses.Courses(db_conn, FastAPICache) -date_range_map = DateMapping.semester_date_mapping(db_conn) +date_range_map = DateMapping.SemesterDateMapping(db_conn) admin_info = AdminInfo.Admin(db_conn) -course_select = CourseSelect.student_course_selection(db_conn) -semester_info = SemesterInfo.semester_info(db_conn) +course_select = CourseSelect.StudentCourseSelection(db_conn) +semester_info = SemesterInfo.SemesterInfo(db_conn) professor_info = All_professors.Professor(db_conn, FastAPICache) +finals_info = Finals.Finals(db_conn) +course_watch_info = CourseWatch.CourseWatchers(db_conn) users = UserModel.User() def is_admin_user(session): @@ -57,7 +56,7 @@ def is_admin_user(session): @app.get('/') @cache(expire=Constants.HOUR_IN_SECONDS, coder=PickleCoder, namespace="API_CACHE") -async def root(request: Request): +async def root(_request: Request): return Response(content='YACS API is Up!',) @app.get('/api') @@ -83,7 +82,7 @@ async def get_classes(request: Request, semester: str or None = None, search: st classes, error = class_info.get_classes_full(semester) return classes if not error else Response(error, status_code=500) return Response(content="missing semester option", status_code=400) - + @app.get('/api/department') @cache(expire=Constants.HOUR_IN_SECONDS, coder=PickleCoder, namespace="API_CACHE") async def get_departments(): @@ -129,28 +128,27 @@ async def get_semesters(): @app.get('/api/semesterInfo') def get_all_semester_info(): all_semester_info, error = class_info.get_all_semester_info() - return all_semester_info if not error else Response(error, status=500) + return all_semester_info if not error else Response(error, status_code=500) @app.get('/api/defaultsemester') -def get_defaultSemester(): +def get_default_semester(): semester, error = admin_info.get_semester_default() print(semester) - return semester if not error else Response(error, status=500) + return semester if not error else Response(error, status_code=500) @app.post('/api/defaultsemesterset') -def set_defaultSemester(semester_set: DefaultSemesterSetPydantic): +def set_default_semester(semester_set: DefaultSemesterSetPydantic): success, error = admin_info.set_semester_default(semester_set.default) if success: return Response(status_code=200) - else: - print(error) - return Response(error.__str__(), status_code=500) + print(error) + return Response(str(error), status_code=500) #Parses the data from the .csv data files @app.post('/api/bulkCourseUpload') -async def uploadHandler( - isPubliclyVisible: str = Form(...), +async def upload_handler( + is_publicly_visible: str = Form(...), file: UploadFile = File(...)): # check for user files print("in process") @@ -161,11 +159,11 @@ async def uploadHandler( # get file contents = await file.read() csv_file = StringIO(contents.decode()) - # update semester infos based on isPubliclyVisible, hiding semester if needed - # is_publicly_visible = request.form.get("isPubliclyVisible", default=False) + # update semester infos based on is_ublicly_visible, hiding semester if needed + # is_publicly_visible = request.form.get("is_publicly_visible", default=False) semesters = pd.read_csv(csv_file)['semester'].unique() for semester in semesters: - semester_info.upsert(semester, isPubliclyVisible) + semester_info.upsert(semester, is_publicly_visible) # Like C, the cursor will be at EOF after full read, so reset to beginning csv_file.seek(0) # Clear out course data of the same semester before population due to @@ -173,28 +171,32 @@ async def uploadHandler( # courses. courses.bulk_delete(semesters=semesters) # Populate DB from CSV - isSuccess, error = courses.populate_from_csv(csv_file) - if (isSuccess): + is_success, error = courses.populate_from_csv(csv_file) + if is_success: + notif_str = "[Notification] New courses are available for the following semester(s): " + for i in semesters: + notif_str = notif_str + i + " " + if is_publicly_visible == "on": + print(notif_str) return Response(status_code=200) - else: - print(error) - return Response(error.__str__(), status_code=500) + print(error) + return Response(str(error), status_code=500) @app.post('/api/bulkProfessorUpload') -async def uploadJSON( - isPubliclyVisible: str = Form(...), - file: UploadFile = File(...)): +async def upload_json( + #_is_publicly_visible: str = Form(...), + file: UploadFile = File(...)): # Check to make sure the user has sent a file if not file: return Response("No file received", 400) - + # Check that we receive a JSON file if file.filename.find('.') == -1 or file.filename.rsplit('.', 1)[1].lower() != 'json': return Response("File must have JSON extension", 400) - + # Get file contents contents = await file.read() - + # Load JSON data try: #convert string to python dict @@ -204,47 +206,81 @@ async def uploadJSON( return Response(f"Invalid JSON data: {str(e)}", 400) # Call populate_from_json method - isSuccess, error = professor_info.populate_from_json(json_data) - if isSuccess: + is_success, error = professor_info.populate_from_json(json_data) + if is_success: print("SUCCESS") return Response(status_code=200) - else: - print("NOT WORKING") - print(error) - return Response(error.__str__(), status_code=500) + print("NOT WORKING") + print(error) + return Response(str(error), status_code=500) + +@app.post('/api/bulkFinalJsonUpload') +async def upload_finals_json(file: UploadFile = File(...), semester: str = Form(...)): + # Check to make sure the user has sent a file + if not file: + return Response("No file received", 400) + + # Check that we receive a JSON file + if file.filename.find('.') == -1 or file.filename.rsplit('.', 1)[1].lower() != 'json': + return Response("File must have JSON extension", 400) + + # Get file contents + contents = await file.read() + + # Load JSON data + try: + #convert string to python dict + json_data = json.loads(contents.decode('utf-8')) + # print(json_data) + except json.JSONDecodeError as e: + return Response(f"Invalid JSON data: {str(e)}", 400) + + # Delete old data + finals_info.remove_semester_finals(semester) + + # Call populate_from_json method + (is_success, error) = finals_info.populate_from_json(json_data, semester) + if is_success: + print("SUCCESS") + print("[Notification] New finals data available for semester: " + semester) + return Response(status_code=200) + print("NOT WORKING") + print(error) + return Response(str(error), status_code=500) @app.post('/api/mapDateRangeToSemesterPart') async def map_date_range_to_semester_part_handler(request: Request): - # This depends on date_start, date_end, and semester_part_name being - # ordered since each field has multiple entries. They should be ordered - # as each dict entry has the value of list. But if it doesn't work, - # look into parameter_storage_class which will change the default - # ImmutableMultiDict class that form uses. https://flask.palletsprojects.com/en/1.0.x/patterns/subclassing/ - form = await request.form() - if (form): - # If checkbox is false, then, by design, it is not included in the form data. - is_publicly_visible = form.get('isPubliclyVisible', default=False) - semester_title = form.get('semesterTitle') - semester_part_names = form.getlist('semester_part_name') - start_dates = form.getlist('date_start') - end_dates = form.getlist('date_end') - if (start_dates and end_dates and semester_part_names and is_publicly_visible is not None and semester_title): - _, error = date_range_map.insert_all(start_dates, end_dates, semester_part_names) - semester_info.upsert(semester_title, is_publicly_visible) - if (not error): - return Response(status_code=200) - else: - return Response(error, status_code=500) - return Response("Did not receive proper form data", status_code=500) + # This depends on date_start, date_end, and semester_part_name being + # ordered since each field has multiple entries. They should be ordered + # as each dict entry has the value of list. But if it doesn't work, + # look into parameter_storage_class which will change the default + # ImmutableMultiDict class that form uses. + # https://flask.palletsprojects.com/en/1.0.x/patterns/subclassing/ + form = await request.form() + if form: + # If checkbox is false, then, by design, it is not included in the form data. + is_publicly_visible = form.get('is_publicly_visible', default=False) + semester_title = form.get('semesterTitle') + semester_part_names = form.getlist('semester_part_name') + start_dates = form.getlist('date_start') + end_dates = form.getlist('date_end') + if start_dates and end_dates and semester_part_names \ + and is_publicly_visible is not None and semester_title: + _, error = date_range_map.insert_all(start_dates, end_dates, semester_part_names) + semester_info.upsert(semester_title, is_publicly_visible) + if not error: + return Response(status_code=200) + return Response(error, status_code=500) + return Response("Did not receive proper form data", status_code=500) @app.get('/api/user/course') async def get_student_courses(request: Request): if 'user' not in request.session: return Response("Not authorized", status_code=403) - courses, error = course_select.get_selection(request.session['user']['user_id']) - return courses if not error else Response(error, status_code=500) + student_courses, error = course_select.get_selection(request.session['user']['user_id']) + return student_courses if not error else Response(error, status_code=500) @app.get('/api/user/{session_id}') @@ -267,7 +303,7 @@ async def delete_user(request: Request, session: UserDeletePydantic): return user_controller.delete_user(session.dict()) @app.put('/api/user') -async def update_user_info(request:Request, user:updateUser): +async def update_user_info(request:Request, user:UpdateUser): if 'user' not in request.session: return Response("Not authorized", status_code=403) @@ -276,7 +312,7 @@ async def update_user_info(request:Request, user:updateUser): @app.post('/api/session') async def log_in(request: Request, credentials: SessionPydantic): session_res = session_controller.add_session(credentials.dict()) - if (session_res['success']): + if session_res['success']: session_data = session_res['content'] # [0] b/c conn.exec uses fetchall() which wraps result in list user = users.get_user(uid=session_data['uid'])[0] @@ -293,30 +329,33 @@ def log_out(request: Request, session: SessionDeletePydantic): return response @app.post('/api/event') -def add_user_event(request: Request, credentials: SessionPydantic): +def add_user_event(_request: Request):# _credentials: SessionPydantic): return Response(status_code=501) @app.post('/api/user/course') async def add_student_course(request: Request, credentials: UserCoursePydantic): if 'user' not in request.session: return Response("Not authorized", status_code=403) - resp, error = course_select.add_selection(credentials.name, credentials.semester, request.session['user']['user_id'], credentials.cid) + _resp, error = course_select.add_selection(credentials.name, credentials.semester, + request.session['user']['user_id'], credentials.cid) return Response(status_code=200) if not error else Response(error, status_code=500) @app.delete('/api/user/course') -async def remove_student_course(request: Request, courseDelete:CourseDeletePydantic): +async def remove_student_course(request: Request, course_delete:CourseDeletePydantic): if 'user' not in request.session: return Response("Not authorized", status_code=403) - resp,error = course_select.remove_selection(courseDelete.name, courseDelete.semester, request.session['user']['user_id'], courseDelete.cid) + _resp,error = course_select.remove_selection(course_delete.name, course_delete.semester, + request.session['user']['user_id'], + course_delete.cid) return Response(status_code=200) if not error else Response(error, status_code=500) @app.get('/api/professor/name/{email}') async def get_professor_name_by_email(email: str): # searches professor's first and last name by email - professorName, error = professor_info.get_professor_name_by_email(email) + professor_name, error = professor_info.get_professor_name_by_email(email) # Return the data as a JSON response - return professorName if not error else Response(content=error, status_code=500) + return professor_name if not error else Response(content=error, status_code=500) @app.get('/api/professor/department/{department}') async def get_professor_from_department(department: str): @@ -330,13 +369,12 @@ async def get_all_professors(): GET /api/professor Cached: 24 Hours """ - professors, error = professor_info.get_all_professors() # replace professor_info with db_manager + professors, error = professor_info.get_all_professors() # replace professor_info with db_manager db_list = [dict(prof) for prof in professors] if professors else [] return db_list if not error else Response(error, status_code = 500) @app.get('/api/professor/phone_number/{email}') async def get_professor_phone_number_by_email(email: str): - phone_number, error = professor_info.get_professor_phone_number_by_email(email) return phone_number if not error else Response(content=error, status_code=500) @@ -350,7 +388,9 @@ async def get_professor_info_by_email(email:str): # async def remove_student_course(request: Request, courseDelete:CourseDeletePydantic): # if 'user' not in request.session: # return Response("Not authorized", status_code=403) -# resp,error = course_select.remove_selection(courseDelete.name, courseDelete.semester, request.session['user']['user_id'], courseDelete.cid) +# resp,error = course_select.remove_selection(courseDelete.name, courseDelete.semester, +# request.session['user']['user_id'], +# courseDelete.cid) # return Response(status_code=200) if not error else Response(error, status_code=500) @app.post('/api/professor/add/{msg}') @@ -366,15 +406,15 @@ async def add_professor(msg:str): # print("dep", info[4]) # print("portfolio_page", info[5]) # print("rcs", id) - + professor, error = professor_info.add_professor(info[0], info[1], info[2], info[3] , info[4], - info[5], info[6], info[7], info[8]) + info[5], info[6]) return professor if not error else Response(error, status_code=500) @app.post('/api/professor/add/test') async def add_test_professor(): - professor, error = professor_info.add_professor("random", "person", "number", "test?@rpi.edu", "CSCI", - "lally 300", "52995") + professor, error = professor_info.add_professor("random", "person", "number", "test?@rpi.edu", + "CSCI", "lally 300", "52995") return professor if not error else Response(content = error, status_code = 500) @app.delete('/api/professor/remove/{email}') @@ -382,3 +422,34 @@ async def remove_professor(email:str): print(email) professor, error = professor_info.remove_professor(email) return professor if not error else Response(str(error), status_code=500) + +@app.delete('/api/semester/{id}') +async def remove_semester(semester_id: str): + print(semester_id) + semester, error = courses.delete_by_semester(semester=semester_id) + return semester if not error else Response(str(error), status_code=500) + +@app.get('/api/getFinalsCRNs') +async def get_finals_crns(): + finals_data = finals_info.get_finals_data() + return finals_data + +@app.get('/api/getFinalDataFromCRN') +async def get_final_data(crn: str): + return finals_info.get_single_final_data(crn) + +@app.post("/api/addCourseWatcher") +async def add_course_watcher(course_crn: str, user_id: int): + course_watch_info.add_watcher(course_crn, user_id) + +@app.delete("/api/removeCourseWatcher") +async def remove_course_watcher(course_crn: str, user_id: int): + course_watch_info.remove_watcher(course_crn, user_id) + +@app.delete("/api/purgeCourseWatchlist") +async def purge_watchlist(): + course_watch_info.purge_course_watchlist() + +@app.get("/api/getCourseWatchers") +async def get_course_watchers(course_crn: str): + return course_watch_info.get_course_watchers(course_crn) diff --git a/src/api/common.py b/src/api/common.py index c1e8898d2..526ce5cf5 100644 --- a/src/api/common.py +++ b/src/api/common.py @@ -21,11 +21,11 @@ def assert_keys_in_form_exist(form, keys): return True -def encrypt(str): +def encrypt(str_in): """ Encrypt the string using SHA256 :param str: string to be encrypted :return: SHA256 encrypted string """ - encrypt_str = hashlib.sha256(str.encode()).hexdigest() + encrypt_str = hashlib.sha256(str_in.encode()).hexdigest() return encrypt_str diff --git a/src/api/conftest.py b/src/api/conftest.py index 76f24849c..c2fa9fe40 100644 --- a/src/api/conftest.py +++ b/src/api/conftest.py @@ -1,17 +1,17 @@ +import sys +import os +import inspect import pytest from fastapi.testclient import TestClient -import sys, os, inspect +from app import app +from tables.database_session import SessionLocal +from tables import Base currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) parentdir = os.path.dirname(currentdir) appdir = os.environ.get("TEST_APP_DIR", parentdir) sys.path.insert(0, appdir) -from app import app -from db.connection import db -from tables.database_session import SessionLocal, DB_PASS -from tables import Base - ### Create the database session and clear tables needed for testing session = SessionLocal() @@ -24,12 +24,12 @@ def client(): yield TestClient(app) @pytest.fixture(scope="session") -def upload(client): +def upload(client_in): multipart_form_data = { 'file': ('test_data.csv', open(appdir + '/tests/test_data.csv', 'rb')), - 'isPubliclyVisible': (None, "on"), + 'is_publicly_visible': (None, "on"), } - return client.post("/api/bulkCourseUpload", + return client_in.post("/api/bulkCourseUpload", files=multipart_form_data) TEST_USER_SIGNUP = { 'email': 'test@email.com', @@ -40,5 +40,5 @@ def upload(client): 'major': 'CSCI' } @pytest.fixture(scope="session") -def post_user(client): - return client.post('/api/user', json=TEST_USER_SIGNUP) +def post_user(client_in): + return client_in.post('/api/user', json=TEST_USER_SIGNUP) diff --git a/src/api/constants.py b/src/api/constants.py index 6e0edf731..44b94e21e 100644 --- a/src/api/constants.py +++ b/src/api/constants.py @@ -1,5 +1,3 @@ -import os - class Constants: """ Global Constants: diff --git a/src/api/controller/session.py b/src/api/controller/session.py index 394092988..c4de8e7ef 100644 --- a/src/api/controller/session.py +++ b/src/api/controller/session.py @@ -1,7 +1,7 @@ +from datetime import datetime from common import * from db.session import Session as SessionModel from db.user import User as UserModel -from datetime import datetime import view.message as msg @@ -43,7 +43,7 @@ def add_session(form): (email, password) = (form['email'], form['password']) users_founded = users.get_user(email=email, password=encrypt(password), enable=True) - if users_founded == None: + if users_founded is None: return msg.error_msg("Failed to validate user information.") if len(users_founded) == 0: @@ -55,7 +55,7 @@ def add_session(form): res = sessions.start_session(new_session_id, uid, start_time) - if res == None: + if res is None: return msg.error_msg("Failed to start a new session.") return msg.success_msg({ diff --git a/src/api/controller/user.py b/src/api/controller/user.py index d26d6b094..2aed1ef8d 100644 --- a/src/api/controller/user.py +++ b/src/api/controller/user.py @@ -13,7 +13,7 @@ def get_user_info(session_id): if session is None or len(session) == 0: return msg.error_msg("Unable to find the session.") - (sessionid, uid, start_time, end_time) = session[0].values() + (_sessionid, uid, _start_time, end_time) = session[0].values() if end_time is not None: return msg.error_msg("This session already canceled.") @@ -23,12 +23,13 @@ def get_user_info(session_id): if len(user) == 0: return msg.error_msg("Unable to find the user") - (uid, name, email, phone, password, major, degree, enable, _, _) = user[0].values() + (uid, name, email, phone, _password, major, degree, _enable, _, _) = user[0].values() - return msg.success_msg({"uid": uid, "name": name, "email": email, "phone": phone, "major": major, "degree": degree}) + return msg.success_msg({"uid": uid, "name": name, "email": email, + "phone": phone, "major": major, "degree": degree}) -def update_user(user:updateUser): +def update_user(user:UpdateUser): users = UserModel() sessions = SessionModel() @@ -40,7 +41,8 @@ def update_user(user:updateUser): major = user.major degree = user.degree - if(name==None or session_id==None or email==None or phone==None or new_password==None or major==None or degree==None): + if name is None or session_id is None or email is None \ + or phone is None or new_password is None or major is None or degree is None: return msg.error_msg("Please check your requests.") if new_password.strip() == "": @@ -57,7 +59,7 @@ def update_user(user:updateUser): if len(session) == 0: return msg.error_msg("Unable to find the session.") - (sessionid, uid, start_time, end_time) = session[0].values() + (_sessionid, uid, _start_time, end_time) = session[0].values() if end_time is not None: return msg.error_msg("This session already canceled.") @@ -95,7 +97,7 @@ def delete_user(form): if len(session) == 0: return msg.error_msg("Unable to find the session.") - (sessionid, uid, start_time, end_time) = session[0].values() + (_sessionid, uid, _start_time, end_time) = session[0].values() if end_time is not None: return msg.error_msg("Expired SessionID") @@ -104,11 +106,11 @@ def delete_user(form): if password.strip() == "": return msg.error_msg("Password cannot be empty.") - findUser = users.get_user(uid=uid, password=encrypt(password), enable=True) - if findUser is None: + find_user = users.get_user(uid=uid, password=encrypt(password), enable=True) + if find_user is None: return msg.error_msg("Failed to find user.") - if len(findUser) == 0: + if len(find_user) == 0: return msg.error_msg("Wrong password.") # Delete User @@ -126,7 +128,8 @@ def delete_user(form): def add_user(form): users = UserModel() - if not assert_keys_in_form_exist(form, ['name', 'email', 'phone', 'password', 'major', 'degree']): + if not assert_keys_in_form_exist(form, ['name', 'email', 'phone', 'password', + 'major', 'degree']): return msg.error_msg("Please check your requests.") name = form['name'] diff --git a/src/api/controller/userevent.py b/src/api/controller/userevent.py index 2dbabab49..9c3710d06 100644 --- a/src/api/controller/userevent.py +++ b/src/api/controller/userevent.py @@ -4,7 +4,7 @@ def add_event(form): - userEvents = UserEvents() + user_events = UserEvents() if not assert_keys_in_form_exist(form, ['uid', 'eventID', 'data', 'createdAt']): return msg.error_msg("Invalid request body.") @@ -15,9 +15,9 @@ def add_event(form): event_data = form['data'] timestamp = form['createdAt'] - res = userEvents.addEvent(uid=uid,eventID=event_id,data=str(event_data),timestamp=timestamp) + res = user_events.add_event(uid=uid,event_id=event_id,data=str(event_data),timestamp=timestamp) - if res == None: + if res is None: return msg.error_msg("Failed to add event.") return msg.success_msg({"msg": "Event added successfully."}) diff --git a/src/api/db/admin.py b/src/api/db/admin.py index 32d8cd988..0be95f327 100644 --- a/src/api/db/admin.py +++ b/src/api/db/admin.py @@ -1,44 +1,41 @@ class Admin: - - def __init__(self, db_conn): - self.db_conn = db_conn - self.interface_name = 'admin_info' - - def get_semester_default(self): - # NOTE: COALESCE takes first non-null vaue from the list - result, error = self.db_conn.execute(""" - SELECT admin.semester FROM admin_settings admin - UNION ALL - SELECT si.semester FROM semester_info si WHERE si.public=true::boolean - LIMIT 1 - """, None, True) - - default_semester = None - - if len(result) == 1: - # parse row - default_semester = result[0]['semester'] ## Only one record in table for admin_settings - - if error: - return (None, error) - else: - return (default_semester, error) - - def set_semester_default(self, semester): - try: - cmd = """ - WITH _ AS (DELETE FROM admin_settings) - INSERT INTO admin_settings(semester) - VALUES(%s) - ON CONFLICT (semester) DO UPDATE SET semester = %s - """ - response, error = self.db_conn.execute(cmd, [semester, semester], False) - - except Exception as e: - # self.db_conn.rollback() - return (False, e) - - if response != None: - return(True, None) - else: - return (False, error) + def __init__(self, db_conn): + self.db_conn = db_conn + self.interface_name = 'admin_info' + + def get_semester_default(self): + # NOTE: COALESCE takes first non-null vaue from the list + result, error = self.db_conn.execute(""" + SELECT admin.semester FROM admin_settings admin + UNION ALL + SELECT si.semester FROM semester_info si WHERE si.public=true::boolean + LIMIT 1 + """, None, True) + + default_semester = None + + if len(result) == 1: + # parse row + default_semester = result[0]['semester'] ## Only one record in table for admin_settings + + if error: + return (None, error) + return (default_semester, error) + + def set_semester_default(self, semester): + try: + cmd = """ + WITH _ AS (DELETE FROM admin_settings) + INSERT INTO admin_settings(semester) + VALUES(%s) + ON CONFLICT (semester) DO UPDATE SET semester = %s + """ + response, error = self.db_conn.execute(cmd, [semester, semester], False) + + except Exception as e: + # self.db_conn.rollback() + return (False, e) + + if response is not None: + return(True, None) + return (False, error) diff --git a/src/api/db/classinfo.py b/src/api/db/classinfo.py index 3829757eb..e27aeed53 100644 --- a/src/api/db/classinfo.py +++ b/src/api/db/classinfo.py @@ -1,431 +1,430 @@ class ClassInfo: - def __init__(self, db_conn): self.db_conn = db_conn self.interface_name = 'class-info' def get_classes_full(self, semester=None): if semester is not None: - classes_by_semester_query = """ - select - c.department, - c.level, - concat(c.department, '-', c.level) as name, - max(c.title) as title, - c.full_title, - c.min_credits, - c.max_credits, - c.description, - c.frequency, - ( - SELECT json_agg(copre.prerequisite) - FROM course_prerequisite copre - WHERE c.department=copre.department - AND c.level=copre.level - ) AS prerequisites, - ( - SELECT json_agg(coco.corequisite) - FROM course_corequisite coco - WHERE c.department=coco.department - AND c.level=coco.level - ) AS corequisites, - c.raw_precoreqs, - c.date_start, - c.date_end, - json_agg( - row_to_json(section.*) - ) sections, - c.semester, - c.school - from - course c - left join - ( - select - c1.crn, - c1.seats_open, - c1.seats_filled, - c1.seats_total, - c1.semester, - c1.min_credits, - c1.max_credits, - max(c1.department) as department, - max(c1.level) as level, - json_agg( - row_to_json(cs.*) - ) sessions - from - course c1 - join course_session cs on - c1.crn = cs.crn and - c1.semester = cs.semester - group by - c1.crn, - c1.semester - ) section - on - c.department = section.department and - c.level = section.level and - c.crn = section.crn - WHERE - c.semester = %s - group by - c.department, - c.level, - c.date_start, - c.date_end, - c.semester, - c.full_title, - c.description, - c.min_credits, - c.max_credits, - c.frequency, - c.raw_precoreqs, - c.school - order by - c.department asc, - c.level asc - """ - return self.db_conn.execute(classes_by_semester_query, [semester], True) + classes_by_semester_query = """ + select + c.department, + c.level, + concat(c.department, '-', c.level) as name, + max(c.title) as title, + c.full_title, + c.min_credits, + c.max_credits, + c.description, + c.frequency, + ( + SELECT json_agg(copre.prerequisite) + FROM course_prerequisite copre + WHERE c.department=copre.department + AND c.level=copre.level + ) AS prerequisites, + ( + SELECT json_agg(coco.corequisite) + FROM course_corequisite coco + WHERE c.department=coco.department + AND c.level=coco.level + ) AS corequisites, + c.raw_precoreqs, + c.date_start, + c.date_end, + json_agg( + row_to_json(section.*) + ) sections, + c.semester, + c.school + from + course c + left join + ( + select + c1.crn, + c1.seats_open, + c1.seats_filled, + c1.seats_total, + c1.semester, + c1.min_credits, + c1.max_credits, + max(c1.department) as department, + max(c1.level) as level, + json_agg( + row_to_json(cs.*) + ) sessions + from + course c1 + join course_session cs on + c1.crn = cs.crn and + c1.semester = cs.semester + group by + c1.crn, + c1.semester + ) section + on + c.department = section.department and + c.level = section.level and + c.crn = section.crn + WHERE + c.semester = %s + group by + c.department, + c.level, + c.date_start, + c.date_end, + c.semester, + c.full_title, + c.description, + c.min_credits, + c.max_credits, + c.frequency, + c.raw_precoreqs, + c.school + order by + c.department asc, + c.level asc + """ + return self.db_conn.execute(classes_by_semester_query, [semester], True) all_classes_query = """ - select - c.department, - c.level, - concat(c.department, '-', c.level) as name, - max(c.title) as title, - c.full_title, - c.min_credits, - c.max_credits, - c.description, - c.frequency, - ( - SELECT json_agg(copre.prerequisite) - FROM course_prerequisite copre - WHERE c.department=copre.department - AND c.level=copre.level - ) AS prerequisites, - ( - SELECT json_agg(coco.corequisite) - FROM course_corequisite coco - WHERE c.department=coco.department - AND c.level=coco.level - ) AS corequisites, - c.raw_precoreqs, - c.date_start, - c.date_end, - json_agg( - row_to_json(section.*) - ) sections, - c.semester, - c.school - from - course c - left join - ( - select - c1.crn, - c1.min_credits, - c1.max_credits, - c1.seats_open, - c1.seats_filled, - c1.seats_total, - c1.semester, - max(c1.department) as department, - max(c1.level) as level, - json_agg( - row_to_json(cs.*) - ) sessions - from - course c1 - join course_session cs on - c1.crn = cs.crn and - c1.semester = cs.semester - group by - c1.crn, - c1.semester - ) section - on - c.department = section.department and - c.level = section.level and - c.crn = section.crn - group by - c.department, - c.level, - c.date_start, - c.date_end, - c.semester, - c.min_credits, - c.max_credits, - c.full_title, - c.description, - c.frequency, - c.raw_precoreqs, - c.school - order by - c.department asc, - c.level asc + select + c.department, + c.level, + concat(c.department, '-', c.level) as name, + max(c.title) as title, + c.full_title, + c.min_credits, + c.max_credits, + c.description, + c.frequency, + ( + SELECT json_agg(copre.prerequisite) + FROM course_prerequisite copre + WHERE c.department=copre.department + AND c.level=copre.level + ) AS prerequisites, + ( + SELECT json_agg(coco.corequisite) + FROM course_corequisite coco + WHERE c.department=coco.department + AND c.level=coco.level + ) AS corequisites, + c.raw_precoreqs, + c.date_start, + c.date_end, + json_agg( + row_to_json(section.*) + ) sections, + c.semester, + c.school + from + course c + left join + ( + select + c1.crn, + c1.min_credits, + c1.max_credits, + c1.seats_open, + c1.seats_filled, + c1.seats_total, + c1.semester, + max(c1.department) as department, + max(c1.level) as level, + json_agg( + row_to_json(cs.*) + ) sessions + from + course c1 + join course_session cs on + c1.crn = cs.crn and + c1.semester = cs.semester + group by + c1.crn, + c1.semester + ) section + on + c.department = section.department and + c.level = section.level and + c.crn = section.crn + group by + c.department, + c.level, + c.date_start, + c.date_end, + c.semester, + c.min_credits, + c.max_credits, + c.full_title, + c.description, + c.frequency, + c.raw_precoreqs, + c.school + order by + c.department asc, + c.level asc """ return self.db_conn.execute(all_classes_query, None, True) def get_departments(self): return self.db_conn.execute(""" - select - distinct(department) - from - course - order by - department asc + select + distinct(department) + from + course + order by + department asc """, None, True) def get_subsemesters(self, semester=None): - if semester is not None: + if semester is not None: + return self.db_conn.execute(""" + select + c.date_start, + c.date_end, + (SELECT semester_part_name FROM semester_date_range sdr WHERE sdr.date_start = c.date_start AND sdr.date_end = c.date_end), + c.semester AS parent_semester_name + from + course c + WHERE + c.semester = %s + group by + c.date_start, + c.date_end, + c.semester + order by + c.date_start asc, + c.date_end desc + """, [semester], True) return self.db_conn.execute(""" - select - c.date_start, - c.date_end, - (SELECT semester_part_name FROM semester_date_range sdr WHERE sdr.date_start = c.date_start AND sdr.date_end = c.date_end), - c.semester AS parent_semester_name - from - course c - WHERE - c.semester = %s - group by - c.date_start, - c.date_end, - c.semester - order by - c.date_start asc, - c.date_end desc - """, [semester], True) - return self.db_conn.execute(""" - select - c.date_start, - c.date_end, - (SELECT semester_part_name FROM semester_date_range sdr WHERE sdr.date_start = c.date_start AND sdr.date_end = c.date_end), - c.semester AS parent_semester_name - from - course c - group by - c.date_start, - c.date_end, - c.semester - order by - c.date_start asc, - c.date_end desc - """, None, True) + select + c.date_start, + c.date_end, + (SELECT semester_part_name FROM semester_date_range sdr WHERE sdr.date_start = c.date_start AND sdr.date_end = c.date_end), + c.semester AS parent_semester_name + from + course c + group by + c.date_start, + c.date_end, + c.semester + order by + c.date_start asc, + c.date_end desc + """, None, True) - def get_semesters(self, includeHidden=False): - if includeHidden: + def get_semesters(self, include_hidden=False): + if include_hidden: + return self.db_conn.execute(""" + select + semester + from + semester_info + """, None, True) return self.db_conn.execute(""" - select - semester - from - semester_info + select + semester + from + semester_info + where + public = true::boolean """, None, True) - return self.db_conn.execute(""" - select - semester - from - semester_info - where - public = true::boolean - """, None, True) def get_all_semester_info(self): - return self.db_conn.execute(""" - SELECT - * - FROM - semester_info - ; - """, None, True) + return self.db_conn.execute(""" + SELECT + * + FROM + semester_info + ; + """, None, True) def get_classes_by_search(self, semester=None, search=None): - if semester is not None: - # # parse search string to a format recognized by to_tsquery - # ts_search = None if search is None else search.strip().replace(' ', '|') - return self.db_conn.execute(""" - WITH ts AS ( - SELECT - c.department, - c.level, - CONCAT(c.department, '-', c.level) AS name, - MAX(c.title) AS title, - c.full_title, - c.min_credits, - c.max_credits, - c.description, - c.frequency, - MAX(c.ts_rank) AS ts_rank, - ( - SELECT JSON_AGG(copre.prerequisite) - FROM course_prerequisite copre - WHERE c.department=copre.department - AND c.level=copre.level - ) AS prerequisites, - ( - SELECT JSON_AGG(coco.corequisite) - FROM course_corequisite coco - WHERE c.department=coco.department - AND c.level=coco.level - ) AS corequisites, - c.raw_precoreqs, - c.date_start, - c.date_end, - JSON_AGG( - row_to_json(section.*) - ) sections, - c.semester - FROM - ( - SELECT - *, - ts_rank_cd(course.tsv, plainto_tsquery(%(search)s)) AS ts_rank - FROM - course - ) AS c - LEFT JOIN - ( - SELECT - c1.crn, - c1.seats_open, - c1.seats_filled, - c1.seats_total, - c1.semester, - MAX(c1.department) AS department, - MAX(c1.level) as level, - JSON_AGG( - row_to_json(cs.*) - ) sessions - FROM - course c1 - JOIN course_session cs on - c1.crn = cs.crn and - c1.semester = cs.semester - GROUP BY - c1.crn, - c1.semester - ) section - ON - c.department = section.department and - c.level = section.level and - c.crn = section.crn - WHERE - c.semester = %(semester)s - AND c.tsv @@ plainto_tsquery(%(search)s) - GROUP BY - c.department, - c.level, - c.date_start, - c.date_end, - c.semester, - c.full_title, - c.min_credits, - c.max_credits, - c.description, - c.frequency, - c.raw_precoreqs - ORDER BY - ts_rank DESC, - department ASC, - level ASC - ) - SELECT * FROM ts - UNION ALL - SELECT * - FROM - ( - SELECT - c.department, - c.level, - CONCAT(c.department, '-', c.level) AS name, - MAX(c.title) AS title, - c.full_title, - c.min_credits, - c.max_credits, - c.description, - c.frequency, - MAX(c.ts_rank) AS ts_rank, - ( - SELECT JSON_AGG(copre.prerequisite) - FROM course_prerequisite copre - WHERE c.department=copre.department - AND c.level=copre.level - ) AS prerequisites, - ( - SELECT JSON_AGG(coco.corequisite) - FROM course_corequisite coco - WHERE c.department=coco.department - AND c.level=coco.level - ) AS corequisites, - c.raw_precoreqs, - c.date_start, - c.date_end, - JSON_AGG( - row_to_json(section.*) - ) sections, - c.semester - FROM - ( - SELECT - *, - ts_rank_cd(course.tsv, plainto_tsquery(%(search)s)) AS ts_rank - FROM - course - ) AS c - LEFT JOIN - ( - SELECT - c1.crn, - c1.seats_open, - c1.seats_filled, - c1.seats_total, - c1.semester, - MAX(c1.department) AS department, - MAX(c1.level) as level, - JSON_AGG( - row_to_json(cs.*) - ) sessions - FROM - course c1 - JOIN course_session cs on - c1.crn = cs.crn and - c1.semester = cs.semester - GROUP BY - c1.crn, - c1.semester - ) section - ON - c.department = section.department and - c.level = section.level and - c.crn = section.crn - WHERE - c.semester = %(semester)s - AND c.full_title ILIKE %(searchAny)s - GROUP BY - c.department, - c.level, - c.date_start, - c.date_end, - c.semester, - c.full_title, - c.min_credits, - c.max_credits, - c.description, - c.frequency, - c.raw_precoreqs - ORDER BY - ts_rank DESC, - department ASC, - level ASC - ) q2 - WHERE NOT EXISTS ( - SELECT * FROM ts - ) - """, { - 'search': search,#ts_search, - 'searchAny': '%' + search + '%', - 'semester': semester - }, True) - return None + if semester is not None: + # # parse search string to a format recognized by to_tsquery + # ts_search = None if search is None else search.strip().replace(' ', '|') + return self.db_conn.execute(""" + WITH ts AS ( + SELECT + c.department, + c.level, + CONCAT(c.department, '-', c.level) AS name, + MAX(c.title) AS title, + c.full_title, + c.min_credits, + c.max_credits, + c.description, + c.frequency, + MAX(c.ts_rank) AS ts_rank, + ( + SELECT JSON_AGG(copre.prerequisite) + FROM course_prerequisite copre + WHERE c.department=copre.department + AND c.level=copre.level + ) AS prerequisites, + ( + SELECT JSON_AGG(coco.corequisite) + FROM course_corequisite coco + WHERE c.department=coco.department + AND c.level=coco.level + ) AS corequisites, + c.raw_precoreqs, + c.date_start, + c.date_end, + JSON_AGG( + row_to_json(section.*) + ) sections, + c.semester + FROM + ( + SELECT + *, + ts_rank_cd(course.tsv, plainto_tsquery(%(search)s)) AS ts_rank + FROM + course + ) AS c + LEFT JOIN + ( + SELECT + c1.crn, + c1.seats_open, + c1.seats_filled, + c1.seats_total, + c1.semester, + MAX(c1.department) AS department, + MAX(c1.level) as level, + JSON_AGG( + row_to_json(cs.*) + ) sessions + FROM + course c1 + JOIN course_session cs on + c1.crn = cs.crn and + c1.semester = cs.semester + GROUP BY + c1.crn, + c1.semester + ) section + ON + c.department = section.department and + c.level = section.level and + c.crn = section.crn + WHERE + c.semester = %(semester)s + AND c.tsv @@ plainto_tsquery(%(search)s) + GROUP BY + c.department, + c.level, + c.date_start, + c.date_end, + c.semester, + c.full_title, + c.min_credits, + c.max_credits, + c.description, + c.frequency, + c.raw_precoreqs + ORDER BY + ts_rank DESC, + department ASC, + level ASC + ) + SELECT * FROM ts + UNION ALL + SELECT * + FROM + ( + SELECT + c.department, + c.level, + CONCAT(c.department, '-', c.level) AS name, + MAX(c.title) AS title, + c.full_title, + c.min_credits, + c.max_credits, + c.description, + c.frequency, + MAX(c.ts_rank) AS ts_rank, + ( + SELECT JSON_AGG(copre.prerequisite) + FROM course_prerequisite copre + WHERE c.department=copre.department + AND c.level=copre.level + ) AS prerequisites, + ( + SELECT JSON_AGG(coco.corequisite) + FROM course_corequisite coco + WHERE c.department=coco.department + AND c.level=coco.level + ) AS corequisites, + c.raw_precoreqs, + c.date_start, + c.date_end, + JSON_AGG( + row_to_json(section.*) + ) sections, + c.semester + FROM + ( + SELECT + *, + ts_rank_cd(course.tsv, plainto_tsquery(%(search)s)) AS ts_rank + FROM + course + ) AS c + LEFT JOIN + ( + SELECT + c1.crn, + c1.seats_open, + c1.seats_filled, + c1.seats_total, + c1.semester, + MAX(c1.department) AS department, + MAX(c1.level) as level, + JSON_AGG( + row_to_json(cs.*) + ) sessions + FROM + course c1 + JOIN course_session cs on + c1.crn = cs.crn and + c1.semester = cs.semester + GROUP BY + c1.crn, + c1.semester + ) section + ON + c.department = section.department and + c.level = section.level and + c.crn = section.crn + WHERE + c.semester = %(semester)s + AND c.full_title ILIKE %(searchAny)s + GROUP BY + c.department, + c.level, + c.date_start, + c.date_end, + c.semester, + c.full_title, + c.min_credits, + c.max_credits, + c.description, + c.frequency, + c.raw_precoreqs + ORDER BY + ts_rank DESC, + department ASC, + level ASC + ) q2 + WHERE NOT EXISTS ( + SELECT * FROM ts + ) + """, { + 'search': search,#ts_search, + 'searchAny': '%' + search + '%', + 'semester': semester + }, True) + return None diff --git a/src/api/db/connection.py b/src/api/db/connection.py index f83e7685b..4a3362716 100644 --- a/src/api/db/connection.py +++ b/src/api/db/connection.py @@ -1,6 +1,6 @@ +import os import psycopg2 import psycopg2.extras -import os # connection details DB_NAME = os.environ.get('DB_NAME', 'yacs') @@ -9,7 +9,10 @@ DB_PORT = os.environ.get('DB_PORT', None) DB_PASS = os.environ.get('DB_PASS', None) -class database(): +class Database(): + def __init__(self): + self.conn = None + def connect(self): self.conn = psycopg2.connect( dbname=DB_NAME, @@ -23,11 +26,11 @@ def connect(self): def close(self): self.conn.close() - def execute(self, sql, args, isSELECT=True): + def execute(self, sql, args, is_select=True): cur = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) ret = None try: - if isSELECT: + if is_select: cur.execute(sql, args) ret = cur.fetchall() else: @@ -47,5 +50,5 @@ def get_connection(self): return self.conn -db = database() +db = Database() db.connect() diff --git a/src/api/db/course_watchers.py b/src/api/db/course_watchers.py new file mode 100644 index 000000000..89198a8c4 --- /dev/null +++ b/src/api/db/course_watchers.py @@ -0,0 +1,82 @@ +""" +Module responsible for interacting with the CourseWatchers table, which is used +to send notifications to users when select courses they choose to watch have any +updates of note +""" +import asyncio +from psycopg2.extras import RealDictCursor + +# https://stackoverflow.com/questions/54839933/importerror-with-from-import-x-on-simple-python-files +if __name__ == "__main__": + import connection +else: + from . import connection + +class CourseWatchers: + def __init__(self, db_conn): + self.db_conn = db_conn + + def add_watcher(self, course_crn, user_id): + """ + Given a course CRN and a user ID, add that user to the list of + users watching the course. If the user is already in that list, + do nothing. If there is not an entry of watchers for the course + CRN, then add it + """ + course_data = self.db_conn.execute("SELECT * FROM course_watch WHERE crn = %(crn)s", + {"crn": course_crn}, True) + if len(course_data[0]) == 0: + self.db_conn.execute("""INSERT INTO course_watch + (crn, watchers) + VALUES + (%(crn)s, '{%(watchers)s}') + ON CONFLICT DO NOTHING;""", + {"crn": course_crn, "watchers": int(user_id)}, False) + return + watcher_list = course_data[0][0]["watchers"].copy() + print(watcher_list) + if user_id not in watcher_list: + watcher_list.append(user_id) + self.db_conn.execute("""UPDATE course_watch + SET + watchers = %(watchers)s + WHERE + crn = %(crn)s;""", + {"crn": course_crn, "watchers": watcher_list}, False) + course_data = self.db_conn.execute("SELECT * FROM course_watch WHERE crn = %(crn)s", + {"crn": course_crn}, True) + print(course_data) + + def remove_watcher(self, course_crn, user_id): + """ + Given a course crn and user id, remove a user from the list + of users watching the course. If the operation results in an + empty list of watchers, leave the empty list in the table. + """ + course_data = self.db_conn.execute("SELECT * FROM course_watch WHERE crn = %(crn)s", + {"crn": course_crn}, True) + watcher_list = course_data[0][0]["watchers"].copy() + print(watcher_list) + if user_id in watcher_list: + watcher_list.remove(user_id) + self.db_conn.execute("""UPDATE course_watch + SET + watchers = %(watchers)s + WHERE + crn = %(crn)s;""", + {"crn": course_crn, "watchers": watcher_list}, False) + course_data = self.db_conn.execute("SELECT * FROM course_watch WHERE crn = %(crn)s", + {"crn": course_crn}, True) + print(course_data) + + def get_course_watchers(self, course_crn): + course_data = self.db_conn.execute("SELECT * FROM course_watch WHERE crn = %(crn)s", + {"crn": course_crn}, True) + return course_data[0][0]["watchers"] + + def purge_course_watchlist(self): + """ + Completely wipe the course watch table. Should be called + if crns change between semesters + """ + self.db_conn.execute("DELETE FROM course_watch", {}, False) diff --git a/src/api/db/courses.py b/src/api/db/courses.py index 89a4363bb..faf7e11d2 100644 --- a/src/api/db/courses.py +++ b/src/api/db/courses.py @@ -1,11 +1,8 @@ -import glob -import os import csv import re -import json -from psycopg2.extras import RealDictCursor from ast import literal_eval import asyncio +from psycopg2.extras import RealDictCursor # https://stackoverflow.com/questions/54839933/importerror-with-from-import-x-on-simple-python-files if __name__ == "__main__": @@ -19,7 +16,7 @@ def __init__(self, db_wrapper, cache): self.db = db_wrapper self.cache = cache - def dayToNum(self, day_char): + def day_to_num(self, day_char): day_map = { 'M': 0, 'T': 1, @@ -30,10 +27,11 @@ def dayToNum(self, day_char): day_num = day_map.get(day_char, -1) if day_num != -1: return day_num + return None - def getDays(self, daySequenceStr): + def get_days(self, day_sequence_str): return set(filter( - lambda day: day, re.split("(?:(M|T|W|R|F))", daySequenceStr))) + lambda day: day, re.split("(?:(M|T|W|R|F))", day_sequence_str))) def delete_by_semester(self, semester): # clear cache so this semester does not come up again @@ -47,7 +45,7 @@ def delete_by_semester(self, semester): COMMIT; """, { "Semester": semester - }, isSELECT=False) + }, is_select=False) def bulk_delete(self, semesters): for semester in semesters: @@ -68,7 +66,7 @@ def populate_from_csv(self, csv_text): for row in reader: try: # course sessions - days = self.getDays(row['course_days_of_the_week']) + days = self.get_days(row['course_days_of_the_week']) for day in days: transaction.execute( """ @@ -101,9 +99,18 @@ def populate_from_csv(self, csv_text): "CRN": row['course_crn'], "Section": row['course_section'], "Semester": row['semester'], - "StartTime": row['course_start_time'] if row['course_start_time'] and not row['course_start_time'].isspace() else None, - "EndTime": row['course_end_time'] if row['course_end_time'] and not row['course_end_time'].isspace() else None, - "WeekDay": self.dayToNum(day) if day and not day.isspace() else None, + "StartTime": row['course_start_time'] \ + if row['course_start_time'] \ + and not row['course_start_time'].isspace() \ + else None, + "EndTime": row['course_end_time'] \ + if row['course_end_time'] \ + and not row['course_end_time'].isspace() \ + else None, + "WeekDay": self.day_to_num(day) \ + if day \ + and not day.isspace() \ + else None, "Location": row['course_location'], "SessionType": row['course_type'], "Instructor": row['course_instructor'] @@ -171,10 +178,19 @@ def populate_from_csv(self, csv_text): "Description": row['description'], # new "Frequency": row['offer_frequency'], # new "FullTitle": row["full_name"], # new - "StartDate": row['course_start_date'] if row['course_start_date'] and not row['course_start_date'].isspace() else None, - "EndDate": row['course_end_date'] if row['course_end_date'] and not row['course_end_date'].isspace() else None, + "StartDate": row['course_start_date'] \ + if row['course_start_date'] \ + and not row['course_start_date'].isspace() \ + else None, + "EndDate": row['course_end_date'] \ + if row['course_end_date'] \ + and not row['course_end_date'].isspace() \ + else None, "Department": row['course_department'], - "Level": row['course_level'] if row['course_level'] and not row['course_level'].isspace() else None, + "Level": row['course_level'] \ + if row['course_level'] \ + and not row['course_level'].isspace() \ + else None, "Title": row['course_name'], "RawPrecoreqText": row['raw_precoreqs'], "School": row['school'], @@ -206,7 +222,10 @@ def populate_from_csv(self, csv_text): """, { "Department": row['course_department'], - "Level": row['course_level'] if row['course_level'] and not row['course_level'].isspace() else None, + "Level": row['course_level'] \ + if row['course_level'] \ + and not row['course_level'].isspace() \ + else None, "Prerequisite": prereq } ) @@ -229,7 +248,8 @@ def populate_from_csv(self, csv_text): """, { "Department": row['course_department'], - "Level": row['course_level'] if row['course_level'] and not row['course_level'].isspace() else None, + "Level": row['course_level'] if row['course_level'] + and not row['course_level'].isspace() else None, "Corequisite": coreq } ) @@ -256,6 +276,6 @@ def clear_cache(self): if __name__ == "__main__": # os.chdir(os.path.abspath("../rpi_data")) # fileNames = glob.glob("*.csv") - csv_text = open('../../../rpi_data/fall-2020.csv', 'r') - courses = Courses(connection.db) - courses.populate_from_csv(csv_text) + with open('../../../rpi_data/fall-2020.csv', 'r', encoding="utf8") as csv_file: + courses = Courses(connection.db, None) + courses.populate_from_csv(csv_file) diff --git a/src/api/db/finals.py b/src/api/db/finals.py new file mode 100644 index 000000000..5b8c25005 --- /dev/null +++ b/src/api/db/finals.py @@ -0,0 +1,144 @@ +""" +Module for managing final exams +""" +import asyncio +from psycopg2.extras import RealDictCursor + +# https://stackoverflow.com/questions/54839933/importerror-with-from-import-x-on-simple-python-files +if __name__ == "__main__": + import connection +else: + from . import connection + +class Finals: + def __init__(self, db_conn): + self.db_conn = db_conn + + def populate_from_json(self, json_data, semester): + """ + Populates the finals table with an array of json data + Any values not present in a data entry are automatically added with None as theit value. + Returns a boolean and string, the bool indicates whether the operation was a success + and the string is the error, if it is not successful. + The following values can be present in each entry: + crn: the crn, must be present + department: the 4-letter department code + level: the course level number + section: the section + title: the title + full_title: the full title + location: the location + day: integer representation of the day the final takes place on: M=0, T=1, W=2, R=3, F=4. + start_time: start time + end_time: the end time + semester: the semester of the final + """ + conn = self.db_conn.get_connection() + + with conn.cursor(cursor_factory=RealDictCursor) as transaction: + try: + # Iterate over each entry in the JSON data + for entry in json_data: + entry["semester"] = semester + for i in ("crn", "department", "level", "section", "title", "full_title", + "location", "day", "start_time", "end_time"): + if i not in entry: + # change behavior here if null data is bad + entry[i] = None + try: + transaction.execute(""" + INSERT INTO finals ( + crn, + department, + level, + section, + title, + full_title, + location, + day, + start_time, + end_time, + semester + ) VALUES ( + %(crn)s, + %(department)s, + %(level)s, + %(section)s, + %(title)s, + %(full_title)s, + %(location)s, + %(day)s, + %(start_time)s, + %(end_time)s, + %(semester)s + ) + ON CONFLICT DO NOTHING; + """, entry) + except Exception as e: + # Roll back the transaction and return the exception if an error occurs + print("THIS IS THE EXCEPTION: ", e) + conn.rollback() + return (False, e) + except ValueError as ve: + # Return an error message if the JSON data is invalid + return (False, f"Invalid JSON data: {str(ve)}") + + # Commit the transaction if all entries are inserted successfully + conn.commit() + + return (True, None) + + def get_finals_data(self): + """ + Returns a list of CRNs for all classes with finals data. Note that some of the + data may be null + """ + finals_data = self.db_conn.execute(""" + SELECT crn FROM finals; + """, {}, True) + ret = [] + for i in finals_data[0]: + ret.append(i["crn"]) + return ret + + def get_single_final_data(self, crn): + """ + Returns the finals data for the crn provided. + """ + return self.db_conn.execute("""SELECT + * FROM finals + WHERE + crn = %(crn)s""", {"crn": crn}, True)[0][0] + + def remove_final(self, crn, section=None): + """ + Removes the final that matches the crn (and if provided, the section) from the table + On error, returns the error + """ + if section is None: + return self.db_conn.execute(""" + DELETE FROM finals + WHERE crn = '%(crn)s' + """, {"crn": crn}, False) + + return self.db_conn.execute(""" + DELETE FROM finals + WHERE crn = '%(crn)s' AND section = '%(day)s' + """, + { + "crn": crn, + "section": section + }, False) + + def remove_semester_finals(self, semester): + """ + Remove finals for a semester, should be called before bulk adding + """ + return self.db_conn.execute(""" + BEGIN TRANSACTION; + DELETE FROM finals + WHERE semester=%(Semester)s; + COMMIT; + """, { + "Semester": semester + }, is_select=False) diff --git a/src/api/db/model.py b/src/api/db/model.py index e2c6d4068..a016950bf 100644 --- a/src/api/db/model.py +++ b/src/api/db/model.py @@ -1,7 +1,6 @@ -import psycopg2 -import db.connection as connection +from db import connection -class Model(object): +class Model(): def __init__(self): self.db = connection.db diff --git a/src/api/db/professor.py b/src/api/db/professor.py index e4a038b8b..1f28af1f1 100644 --- a/src/api/db/professor.py +++ b/src/api/db/professor.py @@ -1,6 +1,5 @@ -import json -from psycopg2.extras import RealDictCursor import asyncio +from psycopg2.extras import RealDictCursor # https://stackoverflow.com/questions/54839933/importerror-with-from-import-x-on-simple-python-files if __name__ == "__main__": @@ -9,7 +8,6 @@ from . import connection class Professor: - def __init__(self, db_conn, cache): self.db_conn = db_conn self.cache = cache @@ -31,8 +29,7 @@ def add_professor(self, name, title, email, phone, dep, portfolio, profile_page) "Portfolio_page": portfolio, "Profile_page": profile_page, }) - else: - return False, "Email cannot be None." + return False, "Email cannot be None." # def add_bulk_professor(self): # # Load the JSON data from a file @@ -61,7 +58,7 @@ def add_professor(self, name, title, email, phone, dep, portfolio, profile_page) def populate_from_json(self, json_data): # Connect to the database conn = self.db_conn.get_connection() - + with conn.cursor(cursor_factory=RealDictCursor) as transaction: try: # Iterate over each entry in the JSON data @@ -126,8 +123,10 @@ def populate_from_json(self, json_data): # Return success status and no error return (True, None) - #removes professor if it exists def remove_professor(self, email): + """ + removes professor if it exists + """ if email is not None: sql = """ DELETE FROM @@ -135,7 +134,7 @@ def remove_professor(self, email): WHERE email = '%s' """ - error = self.db_conn.execute(sql, (email,), False) + _error = self.db_conn.execute(sql, (email,), False) else: return (False, "email cant be none") return (True, None) @@ -161,23 +160,27 @@ def bulk_delete(self,professors): self.clear_cache() return None - # if you expect the SQL statement to return more than one row of data, - # you should pass True as the value for multi. - def get_professor_info_by_email(self, email): - if email is not None: - sql = """ - SELECT - * - FROM - professor - where - email = %s - """ - return self.db_conn.execute(sql, (email,), True) - - #seraches professors who are in a certain department - def get_professors_by_department(self,department): + """ + if you expect the SQL statement to return more than one row of data, + you should pass True as the value for multi. + """ + if email is None: + raise ValueError("Expected email, received None") + sql = """ + SELECT + * + FROM + professor + where + email = %s + """ + return self.db_conn.execute(sql, (email,), True) + + def get_professors_by_department(self,department): + """ + seraches professors who are in a certain department + """ sql = """ select * @@ -189,54 +192,43 @@ def get_professors_by_department(self,department): department, error = self.db_conn.execute(sql, (department,), True) return (department, None) if not error else (False, error) - def get_professor_phone_number_by_email(self, email): - if email is not None: - sql = """ - select - phone_number - from - professor - where - email = %s - """ - info, error = self.db_conn.execute(sql, (email,), True) - return (info, None) if not error else (False, error) - - - #return as a json - def get_all_professors(self): - return self.db_conn.execute(""" + def get_professor_phone_number_by_email(self, email): + if email is None: + raise ValueError("Expected email, received None") + sql = """ + select + phone_number + from + professor + where + email = %s + """ + info, error = self.db_conn.execute(sql, (email,), True) + return (info, None) if not error else (False, error) + + def get_all_professors(self): + """ + return as a json + """ + return self.db_conn.execute(""" SELECT * FROM professor """, None, True) - - #gets prfoessors' phone number by their email - def get_professor_phone_number_by_email(self, email): - if email is not None: - sql = """ - select - phone_number - from - professor - where - email = %s - """ - phone_number, error = self.db_conn.execute(sql, (email,), True) - return (phone_number, None) if not error else (False, error) def get_professor_name_by_email(self, email): - if email is not None: - sql = """ - SELECT - name - FROM - professor - WHERE - email = %s - """ - name, error = self.db_conn.execute(sql, (email,), True) - return (name, None) if not error else (False, error) + if email is None: + raise ValueError("Expected email, received None") + sql = """ + SELECT + name + FROM + professor + WHERE + email = %s + """ + name, error = self.db_conn.execute(sql, (email,), True) + return (name, None) if not error else (False, error) if __name__ == "__main__": - csv_text = open('../../../rpi_data/fall-2020.csv', 'r') - courses = Professor(connection.db) - courses.populate_from_csv(csv_text) + with open('../../../rpi_data/fall-2020.csv', 'r', encoding="utf8") as csv_text: + courses = Professor(connection.db, None) + courses.populate_from_csv(csv_text) diff --git a/src/api/db/semester_date_mapping.py b/src/api/db/semester_date_mapping.py index 2f0740d3e..19061f6f0 100644 --- a/src/api/db/semester_date_mapping.py +++ b/src/api/db/semester_date_mapping.py @@ -1,4 +1,4 @@ -class semester_date_mapping: +class SemesterDateMapping: def __init__(self, db_wrapper): self.db = db_wrapper @@ -13,7 +13,7 @@ def insert(self, date_start, date_end, name): "SemesterPartName": name, "DateStart": date_start, "DateEnd": date_end - }, isSELECT=False) + }, is_select=False) def insert_all(self, start_dates, end_dates, names): if len(start_dates) == len(end_dates) == len(names): @@ -31,8 +31,8 @@ def insert_all(self, start_dates, end_dates, names): "SemesterPartName": names[i], "DateStart": start_dates[i], "DateEnd": end_dates[i] - }, isSELECT=False) - if (error): + }, is_select=False) + if error: return (False, error) return (True, None) - return (False, None) \ No newline at end of file + return (False, None) diff --git a/src/api/db/semester_info.py b/src/api/db/semester_info.py index 5d4222ee7..85cf5f057 100644 --- a/src/api/db/semester_info.py +++ b/src/api/db/semester_info.py @@ -1,9 +1,9 @@ -class semester_info: +class SemesterInfo: def __init__(self, db_wrapper): self.db = db_wrapper - def upsert(self, semester, isPublic): + def upsert(self, semester, is_public): self.db.execute(""" INSERT INTO semester_info (semester, public) VALUES (%(SemesterName)s, %(IsPublic)s) @@ -14,17 +14,17 @@ def upsert(self, semester, isPublic): """, { "SemesterName": semester, - "IsPublic": isPublic - }, isSELECT=False) + "IsPublic": is_public + }, is_select=False) def is_public(self, semester): """ @param: semester name @returns: Boolean indicating if the semester is publicly viewable """ - data, error = self.db.execute(""" + data, _error = self.db.execute(""" SELECT public FROM semester_info WHERE semester=%s LIMIT 1; - """, [semester], isSELECT=True) + """, [semester], is_select=True) if data is not None and len(data) > 0: return data[0]['public'] return False diff --git a/src/api/db/session.py b/src/api/db/session.py index e8173c08b..730916f97 100644 --- a/src/api/db/session.py +++ b/src/api/db/session.py @@ -1,29 +1,30 @@ from datetime import datetime -from db.model import * import uuid +from db.model import Model class Session(Model): - def __init__(self): - super().__init__() def create_session_id(self): return str(uuid.uuid1()) def start_session(self, session, uid, start_time): - sql = """INSERT INTO public.user_session (session_id, user_id, start_time) VALUES (%s,%s,%s);""" + sql = "INSERT INTO public.user_session (session_id, user_id, start_time) VALUES (%s,%s,%s);" args = (session, uid, start_time) return self.db.execute(sql, args, False)[0] def get_session(self, session_id='%'): - sql = """ SELECT session_id, user_id, start_time,end_time + sql = """ SELECT session_id, user_id, start_time, end_time FROM public.user_session - WHERE session_id::text LIKE %s""" + WHERE session_id::text LIKE %s""" arg = (session_id,) return self.db.execute(sql, arg, True)[0] def end_session(self, session_id='%', uid='%', end_time=datetime.utcnow()): - sql = """UPDATE public.user_session SET end_time = %s WHERE session_id::text LIKE %s AND user_id::text LIKE %s;""" + sql = """UPDATE public.user_session + SET end_time = %s WHERE session_id::text + LIKE %s AND user_id::text + LIKE %s;""" args = (end_time, session_id, str(uid)) return self.db.execute(sql, args, False)[0] diff --git a/src/api/db/student_course_selection.py b/src/api/db/student_course_selection.py index 030a36a49..fe3a5088e 100644 --- a/src/api/db/student_course_selection.py +++ b/src/api/db/student_course_selection.py @@ -1,57 +1,57 @@ -class student_course_selection: - def __init__(self, db_conn): - self.db_conn = db_conn +class StudentCourseSelection: + def __init__(self, db_conn): + self.db_conn = db_conn - def add_selection(self, name, sem, uid, cid): - sql = """ - INSERT INTO - student_course_selection (user_id, semester, course_name, crn) - VALUES - (%s, %s, %s, %s) - ON CONFLICT DO NOTHING; - """ - resp, error = self.db_conn.execute(sql, [uid, sem, name, cid], False) - return (True, None) if not error else (False, error) + def add_selection(self, name, sem, uid, cid): + sql = """ + INSERT INTO + student_course_selection (user_id, semester, course_name, crn) + VALUES + (%s, %s, %s, %s) + ON CONFLICT DO NOTHING; + """ + _resp, error = self.db_conn.execute(sql, [uid, sem, name, cid], False) + return (True, None) if not error else (False, error) - def remove_selection(self,name,sem,uid,cid): - if cid is None: - sql = """ - DELETE FROM - student_course_selection - WHERE - user_id = %s AND - semester = %s AND - course_name = %s - """ - resp, error = self.db_conn.execute(sql, [uid, sem, name], False) - else: - sql = """ - DELETE FROM - student_course_selection - WHERE - user_id = %s AND - semester = %s AND - course_name = %s AND - crn = %s + def remove_selection(self,name,sem,uid,cid): + if cid is None: + sql = """ + DELETE FROM + student_course_selection + WHERE + user_id = %s AND + semester = %s AND + course_name = %s + """ + _resp, error = self.db_conn.execute(sql, [uid, sem, name], False) + else: + sql = """ + DELETE FROM + student_course_selection + WHERE + user_id = %s AND + semester = %s AND + course_name = %s AND + crn = %s - """ - resp, error = self.db_conn.execute(sql, [uid, sem, name, cid], False) + """ + _resp, error = self.db_conn.execute(sql, [uid, sem, name, cid], False) - return (True, None) if not error else (False, error) + return (True, None) if not error else (False, error) - def get_selection(self, uid): - sql = """ - select - course_name, - semester, - crn - from - student_course_selection - where - user_id = %s - order by - course_name asc, - crn - """ - courses, error = self.db_conn.execute(sql, [uid], True) - return (courses, None) if not error else (False, error) + def get_selection(self, uid): + sql = """ + select + course_name, + semester, + crn + from + student_course_selection + where + user_id = %s + order by + course_name asc, + crn + """ + courses, error = self.db_conn.execute(sql, [uid], True) + return (courses, None) if not error else (False, error) diff --git a/src/api/db/user.py b/src/api/db/user.py index 699b60a09..7879c105e 100644 --- a/src/api/db/user.py +++ b/src/api/db/user.py @@ -1,20 +1,19 @@ from db.model import * class User(Model): - def __init__(self): - super().__init__() - def get_user(self, uid='%', name='%', email='%', password='%', phone='%', major='%', degree='%', enable=True): - sql = """ SELECT user_id, name, email, phone,password,major,degree,enable,admin,super_admin - FROM public.user_account - WHERE user_id::text LIKE %s AND - name LIKE %s AND - email LIKE %s AND - phone LIKE %s AND - password LIKE %s AND - major LIKE %s AND - degree LIKE %s AND - enable = %s""" + def get_user(self, uid='%', name='%', email='%', password='%', + phone='%', major='%', degree='%', enable=True): + sql ="""SELECT user_id, name, email, phone,password,major,degree,enable,admin,super_admin + FROM public.user_account + WHERE user_id::text LIKE %s AND + name LIKE %s AND + email LIKE %s AND + phone LIKE %s AND + password LIKE %s AND + major LIKE %s AND + degree LIKE %s AND + enable = %s""" args = (str(uid), name, email, phone, password, major, degree, enable) return self.db.execute(sql, args, True)[0] diff --git a/src/api/db/userevent.py b/src/api/db/userevent.py index 960300c54..c4047cbe6 100644 --- a/src/api/db/userevent.py +++ b/src/api/db/userevent.py @@ -1,11 +1,8 @@ -from db.model import * +from db.model import Model class UserEvent(Model): - - def __init__(self): - super().__init__() - - def addEvent(self, uid, eventID, data, timestamp): - sql = """INSERT INTO public.userevents ("eventID", "uid", "data", "createdAt") VALUES (%s, %s, %s, %s)""" - args = (eventID, uid, data, timestamp) + def add_event(self, uid, event_id, data, timestamp): + sql = """INSERT INTO public.userevents ("eventID", "uid", "data", "createdAt") + VALUES (%s, %s, %s, %s)""" + args = (event_id, uid, data, timestamp) return self.db.execute(sql, args, False)[0] diff --git a/src/api/migrations/versions/2024-04-02_testing_finals.py b/src/api/migrations/versions/2024-04-02_testing_finals.py new file mode 100644 index 000000000..5bb86bfa8 --- /dev/null +++ b/src/api/migrations/versions/2024-04-02_testing_finals.py @@ -0,0 +1,40 @@ +"""testing finals + +Revision ID: d48640fdbd48 +Revises: c959c263997f +Create Date: 2024-04-02 20:17:51.135402 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'd48640fdbd48' +down_revision = 'c959c263997f' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('finals', + sa.Column('crn', sa.VARCHAR(length=255), nullable=False), + sa.Column('department', sa.VARCHAR(length=255), nullable=True), + sa.Column('level', sa.INTEGER(), nullable=True), + sa.Column('section', sa.VARCHAR(length=255), nullable=True), + sa.Column('title', sa.VARCHAR(length=255), nullable=True), + sa.Column('full_title', sa.TEXT(), nullable=True), + sa.Column('location', sa.TEXT(), nullable=True), + sa.Column('day', sa.INTEGER(), nullable=True), + sa.Column('start_time', postgresql.TIME(), nullable=True), + sa.Column('end_time', postgresql.TIME(), nullable=True), + sa.PrimaryKeyConstraint('crn') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('finals') + # ### end Alembic commands ### diff --git a/src/api/migrations/versions/2024-04-05_added_semester_details.py b/src/api/migrations/versions/2024-04-05_added_semester_details.py new file mode 100644 index 000000000..0f5051f82 --- /dev/null +++ b/src/api/migrations/versions/2024-04-05_added_semester_details.py @@ -0,0 +1,28 @@ +"""Added semester details + +Revision ID: ef478711822e +Revises: d48640fdbd48 +Create Date: 2024-04-05 20:53:18.924744 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'ef478711822e' +down_revision = 'd48640fdbd48' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('finals', sa.Column('semester', sa.VARCHAR(length=255), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('finals', 'semester') + # ### end Alembic commands ### diff --git a/src/api/migrations/versions/2024-04-12_add_course_watch_table.py b/src/api/migrations/versions/2024-04-12_add_course_watch_table.py new file mode 100644 index 000000000..383c5eaa0 --- /dev/null +++ b/src/api/migrations/versions/2024-04-12_add_course_watch_table.py @@ -0,0 +1,32 @@ +"""Add course watch table + +Revision ID: 9905ff4f7f18 +Revises: ef478711822e +Create Date: 2024-04-12 20:36:30.208020 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '9905ff4f7f18' +down_revision = 'ef478711822e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('course_watch', + sa.Column('crn', sa.VARCHAR(length=255), nullable=False), + sa.Column('watchers', postgresql.ARRAY(sa.INTEGER()), nullable=True), + sa.PrimaryKeyConstraint('crn') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('course_watch') + # ### end Alembic commands ### diff --git a/src/api/requirements.txt b/src/api/requirements.txt index ef22eca2d..57f6b25ca 100644 --- a/src/api/requirements.txt +++ b/src/api/requirements.txt @@ -1,11 +1,11 @@ -fastapi==0.68.0 -psycopg2-binary==2.9.1 -gunicorn==19.9.0 -pandas==1.3.3 -alembic==1.4.3 -pytest==6.2.5 -uvicorn==0.14.0 -python-multipart==0.0.5 -fastapi-cache2==0.1.8 -itsdangerous==2.0.1 -sqlalchemy==1.4.1 +fastapi==0.109.2 +psycopg2-binary==2.9.9 +gunicorn==21.2.0 +pandas==1.5.3 +alembic==1.13.1 +pytest==8.0.0 +uvicorn==0.27.0.post1 +python-multipart==0.0.7 +fastapi-cache2==0.2.1 +itsdangerous==2.1.2 +sqlalchemy==1.4.51 diff --git a/src/api/tables/__init__.py b/src/api/tables/__init__.py index 540c49d23..0784ca6c5 100644 --- a/src/api/tables/__init__.py +++ b/src/api/tables/__init__.py @@ -11,6 +11,8 @@ from .user_event import UserEvent from .user_session import UserSession from .professor import Professor +from .final import Final +from .course_watch import CourseWatch from .database import Base -from .database_session import SessionLocal \ No newline at end of file +from .database_session import SessionLocal diff --git a/src/api/tables/admin_settings.py b/src/api/tables/admin_settings.py index f200d6f42..65113ec18 100644 --- a/src/api/tables/admin_settings.py +++ b/src/api/tables/admin_settings.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, PrimaryKeyConstraint +from sqlalchemy import Column from sqlalchemy.dialects.postgresql import VARCHAR from .database import Base @@ -6,4 +6,4 @@ class AdminSettings(Base): __tablename__ = 'admin_settings' - semester = Column(VARCHAR(length=255), primary_key=True, unique=True) \ No newline at end of file + semester = Column(VARCHAR(length=255), primary_key=True, unique=True) diff --git a/src/api/tables/club_organization.py b/src/api/tables/club_organization.py index 15c6ddf9b..fe1d68f05 100644 --- a/src/api/tables/club_organization.py +++ b/src/api/tables/club_organization.py @@ -13,4 +13,4 @@ class Club(Base): room = Column(VARCHAR(length=255)) days = Column(TSVECTOR) start = Column(DATE) - end = Column(DATE) \ No newline at end of file + end = Column(DATE) diff --git a/src/api/tables/course.py b/src/api/tables/course.py index a249dfbaa..1c250fa28 100644 --- a/src/api/tables/course.py +++ b/src/api/tables/course.py @@ -24,4 +24,4 @@ class Course(Base): seats_open = Column(INTEGER) seats_filled = Column(INTEGER) seats_total = Column(INTEGER) - tsv = Column(TSVECTOR) \ No newline at end of file + tsv = Column(TSVECTOR) diff --git a/src/api/tables/course_corequisite.py b/src/api/tables/course_corequisite.py index e5f29485b..5f4f8ad03 100644 --- a/src/api/tables/course_corequisite.py +++ b/src/api/tables/course_corequisite.py @@ -12,4 +12,4 @@ class CourseCorequisite(Base): __table_args__ = ( PrimaryKeyConstraint('department', 'level', 'corequisite'), - ) \ No newline at end of file + ) diff --git a/src/api/tables/course_prerequisite.py b/src/api/tables/course_prerequisite.py index 9c23dbab9..ebcbb06c4 100644 --- a/src/api/tables/course_prerequisite.py +++ b/src/api/tables/course_prerequisite.py @@ -12,4 +12,4 @@ class CoursePrerequisite(Base): __table_args__ = ( PrimaryKeyConstraint('department', 'level', 'prerequisite'), - ) \ No newline at end of file + ) diff --git a/src/api/tables/course_watch.py b/src/api/tables/course_watch.py new file mode 100644 index 000000000..743e51e12 --- /dev/null +++ b/src/api/tables/course_watch.py @@ -0,0 +1,10 @@ +from sqlalchemy import Column +from sqlalchemy.dialects.postgresql import VARCHAR, ARRAY, INTEGER + +from .database import Base + +class CourseWatch(Base): + __tablename__ = "course_watch" + + crn = Column(VARCHAR(length=255), primary_key=True) + watchers = Column(ARRAY(INTEGER)) diff --git a/src/api/tables/database.py b/src/api/tables/database.py index 7c2377aec..860e54258 100644 --- a/src/api/tables/database.py +++ b/src/api/tables/database.py @@ -1,3 +1,3 @@ from sqlalchemy.ext.declarative import declarative_base -Base = declarative_base() \ No newline at end of file +Base = declarative_base() diff --git a/src/api/tables/database_session.py b/src/api/tables/database_session.py index 8062f3b00..7473d4fec 100644 --- a/src/api/tables/database_session.py +++ b/src/api/tables/database_session.py @@ -15,22 +15,22 @@ if __name__=="__main__": # Handly for waiting until database is online. # Will wait 1 second after each connection attempt, - # total 5 attempts. Throws an exception if all tries + # total 5 attempts. Throws an exception if all tries # fail. import time - is_online = False + IS_ONLINE = False for i in range(5): try: db = SessionLocal() db.execute("SELECT 1") - is_online = True + IS_ONLINE = True break except Exception as err: print(err) - + time.sleep(1) - - if not is_online: - raise Exception("Database not connected") \ No newline at end of file + + if not IS_ONLINE: + raise Exception("Database not connected") diff --git a/src/api/tables/event.py b/src/api/tables/event.py index 72d6b2858..a9c31af79 100644 --- a/src/api/tables/event.py +++ b/src/api/tables/event.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, PrimaryKeyConstraint +from sqlalchemy import Column from sqlalchemy.dialects.postgresql import INTEGER, VARCHAR from .database import Base @@ -7,4 +7,4 @@ class Event(Base): __tablename__ = "event" event_id = Column(INTEGER, primary_key=True) - description = Column(VARCHAR(length=255)) \ No newline at end of file + description = Column(VARCHAR(length=255)) diff --git a/src/api/tables/final.py b/src/api/tables/final.py new file mode 100644 index 000000000..240bfded8 --- /dev/null +++ b/src/api/tables/final.py @@ -0,0 +1,19 @@ +from sqlalchemy import Column +from sqlalchemy.dialects.postgresql import TEXT, INTEGER, VARCHAR, TIME + +from .database import Base + +class Final(Base): + __tablename__ = "finals" + + crn = Column(VARCHAR(length=255), primary_key=True) + department = Column(VARCHAR(length=255)) + level = Column(INTEGER) + section = Column(VARCHAR(length=255)) + title = Column(VARCHAR(length=255)) + full_title = Column(TEXT) + location = Column(TEXT) + day = Column(INTEGER) + start_time = Column(TIME) + end_time = Column(TIME) + semester = Column(VARCHAR(length=255)) diff --git a/src/api/tables/pathways.py b/src/api/tables/pathways.py index 2a3974ea0..2a4002d23 100644 --- a/src/api/tables/pathways.py +++ b/src/api/tables/pathways.py @@ -9,10 +9,10 @@ class Pathway(Base): category_name = Column(VARCHAR(length=255)) pathways_name = Column(VARCHAR(length=255)) description = TEXT - required_courses = Column(TSVECTOR, bool = False) + required_courses = Column(TSVECTOR, bool = False) #description and list of courses for reuqired_courses #foreign key for courses courses = Column(TSVECTOR, ForeignKey("course.crn")) compatible_minor = Column(TSVECTOR) - # [Arts/Design[Creative Design, Grahpic], ] \ No newline at end of file + # [Arts/Design[Creative Design, Grahpic], ] diff --git a/src/api/tables/semester_date_range.py b/src/api/tables/semester_date_range.py index e83af9284..98630845b 100644 --- a/src/api/tables/semester_date_range.py +++ b/src/api/tables/semester_date_range.py @@ -12,4 +12,4 @@ class SemesterDateRange(Base): __table_args__ = ( PrimaryKeyConstraint('date_start', 'date_end'), - ) \ No newline at end of file + ) diff --git a/src/api/tables/semester_info.py b/src/api/tables/semester_info.py index 74d49678a..71b563467 100644 --- a/src/api/tables/semester_info.py +++ b/src/api/tables/semester_info.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, PrimaryKeyConstraint +from sqlalchemy import Column from sqlalchemy.dialects.postgresql import VARCHAR, BOOLEAN from .database import Base diff --git a/src/api/tables/student_course_selection.py b/src/api/tables/student_course_selection.py index 711914fce..8f5b6e246 100644 --- a/src/api/tables/student_course_selection.py +++ b/src/api/tables/student_course_selection.py @@ -13,4 +13,4 @@ class StudentCourseSelection(Base): __table_args__ = ( PrimaryKeyConstraint('user_id', 'semester', 'course_name', 'crn'), - ) \ No newline at end of file + ) diff --git a/src/api/tables/user_account.py b/src/api/tables/user_account.py index c107ae0c9..5825dff6b 100644 --- a/src/api/tables/user_account.py +++ b/src/api/tables/user_account.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, PrimaryKeyConstraint +from sqlalchemy import Column from sqlalchemy.dialects.postgresql import TEXT, INTEGER, BOOLEAN from .database import Base @@ -15,4 +15,4 @@ class UserAccount(Base): degree = Column(TEXT) enable = Column(BOOLEAN, default=True) admin = Column(BOOLEAN, default=False) - super_admin = Column(BOOLEAN, default=False) \ No newline at end of file + super_admin = Column(BOOLEAN, default=False) diff --git a/src/api/tables/user_event.py b/src/api/tables/user_event.py index acc448945..b44dd8ada 100644 --- a/src/api/tables/user_event.py +++ b/src/api/tables/user_event.py @@ -13,4 +13,4 @@ class UserEvent(Base): __table_args__ = ( PrimaryKeyConstraint('event_id', 'user_id'), - ) \ No newline at end of file + ) diff --git a/src/api/tables/user_session.py b/src/api/tables/user_session.py index 252bfee7a..2a2316beb 100644 --- a/src/api/tables/user_session.py +++ b/src/api/tables/user_session.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, PrimaryKeyConstraint +from sqlalchemy import Column from sqlalchemy.dialects.postgresql import INTEGER, TIMESTAMP, UUID from .database import Base @@ -9,4 +9,4 @@ class UserSession(Base): session_id = Column(UUID, primary_key=True) user_id = Column(INTEGER, nullable=False) start_time = Column(TIMESTAMP(timezone=True)) - end_time = Column(TIMESTAMP(timezone=True)) \ No newline at end of file + end_time = Column(TIMESTAMP(timezone=True)) diff --git a/src/api/tests/test_bulk_upload.py b/src/api/tests/test_bulk_upload.py index 8d76b2d0c..479982984 100644 --- a/src/api/tests/test_bulk_upload.py +++ b/src/api/tests/test_bulk_upload.py @@ -1,6 +1,7 @@ +import os +import inspect import pytest from fastapi.testclient import TestClient -import os, inspect current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) appdir = os.environ.get("TEST_APP_DIR", os.path.dirname(current_dir)) @@ -28,7 +29,7 @@ def test_bulk_upload_no_file(client: TestClient): ''' multipart_form_data = { 'file': ('test_data.csv'), - 'isPubliclyVisible': (None, "on"), + 'is_publicly_visible': (None, "on"), } r = client.post("/api/bulkCourseUpload", files=multipart_form_data) @@ -41,7 +42,7 @@ def test_bulk_upload_wrong_file_extension(client: TestClient): ''' multipart_form_data = { 'file': ('test_bulk_upload.py', open(appdir + '/tests/test_bulk_upload.py', 'rb')), - 'isPubliclyVisible': (None, "on"), + 'is_publicly_visible': (None, "on"), } r = client.post("/api/bulkCourseUpload", files=multipart_form_data) diff --git a/src/api/tests/test_class.py b/src/api/tests/test_class.py index 0aea83332..f242fdc13 100644 --- a/src/api/tests/test_class.py +++ b/src/api/tests/test_class.py @@ -1,4 +1,4 @@ -from fastapi.testclient import TestClient +from fastapi.testclient import TestClient import pytest #NOTE: Currently unable to test for non-public semesters access if @@ -20,10 +20,10 @@ def test_wrong_semester(upload, client: TestClient): params = {'semester' : 'RANDOM'} r = client.get("/api/class", params = params) assert r.status_code == 401 - assert (r.text == "Semester isn't available") + assert r.text == "Semester isn't available" @pytest.mark.testclient def test_no_args(upload, client: TestClient): assert upload.status_code == 200 r = client.get("/api/class") - assert(r.text == "missing semester option") + assert r.text == "missing semester option" diff --git a/src/api/tests/test_default_semester.py b/src/api/tests/test_default_semester.py index b216b5670..b8212832e 100644 --- a/src/api/tests/test_default_semester.py +++ b/src/api/tests/test_default_semester.py @@ -2,8 +2,8 @@ from fastapi.testclient import TestClient @pytest.mark.testclient -def test_default_semester_success(client: TestClient, upload): +def test_default_semester_success(client: TestClient, _upload): r = client.get("/api/defaultsemester") data = r.json() - assert data == "SUMMER 2020" \ No newline at end of file + assert data == "SUMMER 2020" diff --git a/src/api/tests/test_default_semester_set.py b/src/api/tests/test_default_semester_set.py index be9da635f..613e1a638 100644 --- a/src/api/tests/test_default_semester_set.py +++ b/src/api/tests/test_default_semester_set.py @@ -7,8 +7,8 @@ def test_default_semester_set(client: TestClient): assert r.status_code == 200 r = client.get("/api/defaultsemester") data = r.json() - assert data == "SUMMER 2020" - + assert data == "SUMMER 2020" + r = client.post('/api/defaultsemesterset', json = {'default' :'SPRING 2020'}) assert r.status_code == 200 r = client.get("/api/defaultsemester") diff --git a/src/api/tests/test_department.py b/src/api/tests/test_department.py index 857b55df9..55df1dc7d 100644 --- a/src/api/tests/test_department.py +++ b/src/api/tests/test_department.py @@ -1,4 +1,4 @@ -from fastapi.testclient import TestClient +from fastapi.testclient import TestClient import pytest @pytest.mark.testclient diff --git a/src/api/tests/test_map_date_range.py b/src/api/tests/test_map_date_range.py index a85af29b9..3c6c6a9c7 100644 --- a/src/api/tests/test_map_date_range.py +++ b/src/api/tests/test_map_date_range.py @@ -4,11 +4,11 @@ @pytest.mark.testclient def test_map_date_range(client: TestClient): r = client.post('/api/mapDateRangeToSemesterPart', data=[ - ('semesterTitle', 'SUMMER 2020'), ('isPubliclyVisible', 'true'), - ('date_start', '2020-05-26'), ('date_start', '2020-05-26'), - ('date_start', '2020-07-13'), ('date_end', '2020-08-21'), - ('date_end', '2020-07-10'), ('date_end', '2020-08-21'), - ('semester_part_name', '5/26 - 8/22'), ('semester_part_name', '5/26 - 7/10'), + ('semesterTitle', 'SUMMER 2020'), ('is_publicly_visible', 'true'), + ('date_start', '2020-05-26'), ('date_start', '2020-05-26'), + ('date_start', '2020-07-13'), ('date_end', '2020-08-21'), + ('date_end', '2020-07-10'), ('date_end', '2020-08-21'), + ('semester_part_name', '5/26 - 8/22'), ('semester_part_name', '5/26 - 7/10'), ('semester_part_name', '7/13 - 8/21')]) assert r.status_code == 200 @@ -16,9 +16,9 @@ def test_map_date_range(client: TestClient): @pytest.mark.testclient def test_map_date_range_failure(client: TestClient): r = client.post('/api/mapDateRangeToSemesterPart', data=[ - ('semesterTitle', 'SUMMER 2020'), ('isPubliclyVisible', 'true'), - ('date_start', '2020-05-26'), ('date_start', '2020-05-26'), - ('date_start', '2020-07-13'), ('date_end', '2020-08-21'), + ('semesterTitle', 'SUMMER 2020'), ('is_publicly_visible', 'true'), + ('date_start', '2020-05-26'), ('date_start', '2020-05-26'), + ('date_start', '2020-07-13'), ('date_end', '2020-08-21'), ('date_end', '2020-07-10'), ('date_end', '2020-08-21')]) assert r.status_code == 500 diff --git a/src/api/tests/test_root.py b/src/api/tests/test_root.py index 9b0bca800..c5c459fe0 100644 --- a/src/api/tests/test_root.py +++ b/src/api/tests/test_root.py @@ -4,9 +4,9 @@ @pytest.mark.testclient def test_root(client: TestClient): r = client.get("/") - assert(r.text == "YACS API is Up!") + assert r.text == "YACS API is Up!" @pytest.mark.testclient def test_api(client: TestClient): r = client.get("/api") - assert(r.text == "wow") + assert r.text == "wow" diff --git a/src/api/tests/test_semester.py b/src/api/tests/test_semester.py index 32f6bc6c3..9615da14d 100644 --- a/src/api/tests/test_semester.py +++ b/src/api/tests/test_semester.py @@ -2,13 +2,14 @@ from fastapi.testclient import TestClient @pytest.mark.testclient -def test_semester(upload, client: TestClient): +def test_semester(_upload, client: TestClient): """ semester endpoint should get all of the semesters in the upload - in the case of our test data, we should be getting get 2 semesters: "SUMMER 2020" and "SPRING 2020" + in the case of our test data, we should be getting get 2 semesters: + "SUMMER 2020" and "SPRING 2020" """ r = client.get("/api/semester") data = r.json() assert len(data) == 2 assert data[0]["semester"] == "SUMMER 2020" - assert data[1]["semester"] == "SPRING 2020" \ No newline at end of file + assert data[1]["semester"] == "SPRING 2020" diff --git a/src/api/tests/test_semester_info.py b/src/api/tests/test_semester_info.py index 8dd28c129..05b68f710 100644 --- a/src/api/tests/test_semester_info.py +++ b/src/api/tests/test_semester_info.py @@ -2,7 +2,7 @@ from fastapi.testclient import TestClient @pytest.mark.testclient -def test_semester_info_success(client: TestClient, upload): +def test_semester_info_success(client: TestClient, _upload): r = client.get("/api/semesterInfo") data = r.json() assert len(data) == 2 @@ -11,5 +11,5 @@ def test_semester_info_success(client: TestClient, upload): assert "SUMMER 2020" in semesters assert "SPRING 2020" in semesters - assert data[0]["public"] == True - assert data[1]["public"] == True \ No newline at end of file + assert data[0]["public"] + assert data[1]["public"] diff --git a/src/api/tests/test_session.py b/src/api/tests/test_session.py index 4c4c9f96a..64fb830a4 100644 --- a/src/api/tests/test_session.py +++ b/src/api/tests/test_session.py @@ -5,7 +5,7 @@ 'password': '123456' } @pytest.mark.testclient -def test_session_post_success(post_user, client: TestClient): +def test_session_post_success(_post_user, client: TestClient): ''' Test session post with valid credentials ''' @@ -30,15 +30,15 @@ def test_session_post_failure(client: TestClient): assert data['content'] is None @pytest.mark.testclient -def test_session_delete_success(post_user, client: TestClient): +def test_session_delete_success(_post_user, client: TestClient): ''' Test session delete with valid input ''' sess = client.post("/api/session", json=TEST_USER).json() - sessID = sess['content']['sessionID'] - r = client.delete('/api/session', json={'sessionID': sessID}) + sess_id = sess['content']['sessionID'] + r = client.delete('/api/session', json={'sessionID': sess_id}) assert r.status_code == 200 - assert r.json()['success'] == True + assert r.json()['success'] @pytest.mark.testclient def test_session_delete_failure(client: TestClient): @@ -47,4 +47,4 @@ def test_session_delete_failure(client: TestClient): ''' r = client.delete('/api/session', json={'sessionID': 'not-a-session-id'}) assert r.status_code == 200 - assert r.json()['success'] == False + assert not r.json()['success'] diff --git a/src/api/tests/test_subsemester.py b/src/api/tests/test_subsemester.py index bd846bd3e..d88872b05 100644 --- a/src/api/tests/test_subsemester.py +++ b/src/api/tests/test_subsemester.py @@ -2,7 +2,7 @@ from fastapi.testclient import TestClient @pytest.mark.testclient -def test_subsemeseter_spring2020(client: TestClient, upload): +def test_subsemeseter_spring2020(client: TestClient, _upload): """ when subsemester endpoint is given an input such as Spring 2020 it should only return data for that subsemester i.e. data where the parent semester name @@ -16,10 +16,10 @@ def test_subsemeseter_spring2020(client: TestClient, upload): assert data[0]["parent_semester_name"] == "SPRING 2020" @pytest.mark.testclient -def test_subsemester_nosemester(client, upload): +def test_subsemester_nosemester(client, _upload): """ - when no subsemester is taken as input the subsemester endpoint should return all of the subsemesters - contained in the upload data. + when no subsemester is taken as input the subsemester endpoint should + return all of the subsemesters contained in the upload data. Specifically there should be 4 total returned semesters """ r = client.get("/api/subsemester") diff --git a/src/api/tests/test_user.py b/src/api/tests/test_user.py index 0635e05ca..626c7859f 100644 --- a/src/api/tests/test_user.py +++ b/src/api/tests/test_user.py @@ -31,31 +31,34 @@ @pytest.mark.testclient -def test_user_post_success(post_user, client: TestClient): +def test_user_post_success(_post_user, client: TestClient): ''' add a valid new user into the session ''' - r = client.post("/api/user", json= {"name": "test2", "email":"test12@gmail.com","phone": "", "password":"test123", "major":"", "degree":""} ) + r = client.post("/api/user", json= {"name": "test2", "email":"test12@gmail.com", + "phone": "", "password":"test123", "major":"", "degree":""}) data = r.json() assert r.status_code == 200 assert data['content'] is not None assert data['content']['msg'] == "User added successfully." r = client.post('/api/session', json={"email":"test12@gmail.com", "password":"test123"}) assert r.status_code == 200 - client.delete("/api/user", json={"sessionID":r.json()['content']['sessionID'], 'password':'test123'}) + client.delete("/api/user", json={"sessionID":r.json()['content']['sessionID'], + 'password':'test123'}) @pytest.mark.testclient -def test_user_post_failure(post_user, client: TestClient): +def test_user_post_failure(_post_user, client: TestClient): ''' add a invalid new user into the session ''' - r = client.post("/api/user", json= {"name": "test1", "email":"test1","phone": "", "password":"123", "major":"", "degree":""}) + r = client.post("/api/user", json= {"name": "test1", "email":"test1","phone": "", + "password":"123", "major":"", "degree":""}) data = r.json() assert data['content'] is None assert r.status_code == 200 - + @pytest.mark.testclient -def test_user_delete_success(post_user, client: TestClient): +def test_user_delete_success(_post_user, client: TestClient): ''' delete a valid user in the session ''' @@ -69,7 +72,7 @@ def test_user_delete_success(post_user, client: TestClient): data = r.json() @pytest.mark.testclient -def test_user_delete_failure(post_user, client: TestClient): +def test_user_delete_failure(_post_user, client: TestClient): ''' delete a not exist user in the session ''' @@ -82,19 +85,19 @@ def test_user_delete_failure(post_user, client: TestClient): assert data2['errMsg'] == "Wrong password." @pytest.mark.testclient -def test_user_delete_failure2(post_user, client: TestClient): +def test_user_delete_failure2(_post_user, client: TestClient): ''' delete the session, then try to delete user ''' sess = client.post("/api/session", json=TEST_USER).json() - sessID = sess['content']['sessionID'] - r = client.delete('/api/session', json={'sessionID': sessID}) + sess_id = sess['content']['sessionID'] + r = client.delete('/api/session', json={'sessionID': sess_id}) assert r.status_code == 200 - r1 = client.delete("/api/user", json= {"sessionID": sessID, "password": "12345"}) + r1 = client.delete("/api/user", json= {"sessionID": sess_id, "password": "12345"}) assert r1.status_code == 403 @pytest.mark.testclient -def test_get_user_success(client: TestClient, post_user): +def test_get_user_success(client: TestClient, _post_user): ''' Test user get by using /api/session and TEST_USER_SIGNUP. ''' @@ -116,7 +119,7 @@ def test_get_user_success(client: TestClient, post_user): client.delete("/api/session", json={'sessionID': sessionid}) @pytest.mark.testclient -def test_get_user_failed(client: TestClient,post_user): +def test_get_user_failed(client: TestClient,_post_user): ''' Test user get with invalid sessionID ''' @@ -136,7 +139,7 @@ def test_get_user_failed(client: TestClient,post_user): client.delete("/api/session", json={'sessionID': sessionid}) @pytest.mark.testclient -def test_get_user_after_session_closed(client: TestClient,post_user): +def test_get_user_after_session_closed(client: TestClient,_post_user): ''' Test user get by using /api/session and TEST_USER_SIGNUP after session is closed. ''' @@ -154,7 +157,7 @@ def test_get_user_after_session_closed(client: TestClient,post_user): @pytest.mark.testclient -def test_put_user_success(client:TestClient,post_user): +def test_put_user_success(client:TestClient,_post_user): ''' Test user put by changing TEST_USER_SIGNUP to TEST_USER_SIGNUP2 compare the user information with TEST_USER_SIGNUP2 @@ -195,7 +198,7 @@ def test_put_user_success(client:TestClient,post_user): @pytest.mark.testclient -def test_put_user_after_session_closed(client:TestClient,post_user): +def test_put_user_after_session_closed(client:TestClient,_post_user): ''' Test user put by changing TEST_USER_SIGNUP to TEST_USER_SIGNUP2 after session is closed @@ -212,4 +215,4 @@ def test_put_user_after_session_closed(client:TestClient,post_user): TEST_USER_SIGNUP2['sessionID'] = data['content']['sessionID'] r=client.delete("/api/session", json={'sessionID': sessionid}) r = client.put("/api/user",json = TEST_USER_SIGNUP2) - assert r.status_code == 403 \ No newline at end of file + assert r.status_code == 403 diff --git a/src/api/tests/test_user_course.py b/src/api/tests/test_user_course.py index 5f895785b..7fa89a164 100644 --- a/src/api/tests/test_user_course.py +++ b/src/api/tests/test_user_course.py @@ -18,7 +18,7 @@ @pytest.mark.testclient -def test_user_course_post_success(post_user, client: TestClient): +def test_user_course_post_success(_post_user, client: TestClient): ''' Test user course post by comparing it to user get course ''' @@ -35,16 +35,16 @@ def test_user_course_post_success(post_user, client: TestClient): assert d.status_code == 200 @pytest.mark.testclient -def test_user_course_post_failure(post_user, client: TestClient): +def test_user_course_post_failure(_post_user, client: TestClient): ''' Test user course post with invalid parameter ''' - TEST_INVALID_USER_COURSE = {} + test_invalid_user_course = {} sess = client.post("/api/session", json=TEST_USER).json() - sessID = sess['content']['sessionID'] - r = client.post("/api/user/course", json=TEST_INVALID_USER_COURSE) + sess_id = sess['content']['sessionID'] + r = client.post("/api/user/course", json=test_invalid_user_course) assert r.status_code == 422 - d = client.delete('/api/session', json={'sessionID': sessID}) + d = client.delete('/api/session', json={'sessionID': sess_id}) assert d.status_code == 200 @pytest.mark.testclient @@ -58,7 +58,7 @@ def test_course_post_not_authorized(client: TestClient): assert r.status_code == 403 @pytest.mark.testclient -def test_user_course_get_success(post_user, client: TestClient): +def test_user_course_get_success(_post_user, client: TestClient): ''' Test user course get success ''' @@ -82,7 +82,7 @@ def test_user_course_get_failure(client: TestClient): assert r.status_code == 403 @pytest.mark.testclient -def test_user_course_delete_success(post_user, client: TestClient): +def test_user_course_delete_success(_post_user, client: TestClient): ''' Test user/course delete success ''' @@ -93,7 +93,8 @@ def test_user_course_delete_success(post_user, client: TestClient): r = client.get("/api/user/course") assert r.status_code == 200 data = r.json() - db_course = {'course_name': TEST_COURSE['name'], 'crn': TEST_COURSE['cid'], 'semester': TEST_COURSE['semester']} + db_course = {'course_name': TEST_COURSE['name'], 'crn': TEST_COURSE['cid'], + 'semester': TEST_COURSE['semester']} assert db_course in data x = client.delete("/api/user/course", json = TEST_COURSE) diff --git a/src/web/src/components/EditSemesterDateNameBinding.vue b/src/web/src/components/EditSemesterDateNameBinding.vue index cc310ba3d..3caea6b18 100644 --- a/src/web/src/components/EditSemesterDateNameBinding.vue +++ b/src/web/src/components/EditSemesterDateNameBinding.vue @@ -13,7 +13,7 @@

{{ semesterTitle }}

diff --git a/src/web/src/pages/UploadCsv.vue b/src/web/src/pages/UploadCsv.vue index 3aca901d3..c7ff82327 100644 --- a/src/web/src/pages/UploadCsv.vue +++ b/src/web/src/pages/UploadCsv.vue @@ -17,7 +17,7 @@
diff --git a/src/web/src/pages/UploadJson.vue b/src/web/src/pages/UploadJson.vue index 7077a2716..16f9b9555 100644 --- a/src/web/src/pages/UploadJson.vue +++ b/src/web/src/pages/UploadJson.vue @@ -15,7 +15,7 @@
diff --git a/tests/api/db/test_semester_date_mapping.py b/tests/api/db/test_semester_date_mapping.py index 8051755b8..0d34c1573 100644 --- a/tests/api/db/test_semester_date_mapping.py +++ b/tests/api/db/test_semester_date_mapping.py @@ -1,5 +1,5 @@ from src.api.db.classinfo import ClassInfo -from src.api.db.semester_date_mapping import semester_date_mapping as SemesterDateMapping +from src.api.db.semester_date_mapping import SemesterDateMapping as SemesterDateMapping from tests.test_data import TestData def test_semester_date_mapping_insert(test_data: TestData, semester_date_mapping: SemesterDateMapping, class_info: ClassInfo): diff --git a/tests/test_data.py b/tests/test_data.py index f427d72e7..114a13034 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -6,7 +6,7 @@ from typing import Dict, Set -from src.api.db.semester_info import semester_info as SemesterInfo +from src.api.db.semester_info import SemesterInfo as SemesterInfo from src.api.db.courses import Courses from tests.mock_cache import MockCache