Merge pull request #118 from DIA-NZ/feature/117_copy_h1_to_h3_profile…

…_overrides

DDL changes: #113, #114 and #117

docs/guides/release-notes.rst

@@ -46,8 +46,9 @@ following sections:
 Changes since 2.0.2
 ===================
 
-This is a placeholder for changes since the official *2.0.1* release. Please
-add notes here for changes and fixes as they are released into the master branch.
+-  Fixed: the check constraint definitions in wct-schema-mysql.sql contained errors that 
+   have become an issue in recent versions of MySQL/MariaDB, since they actually enforce check 
+   constraints.
 
 
 2.0.2

wct-core/db/latest/sql/wct-schema-mysql.sql

@@ -132,8 +132,8 @@ alter table DB_WCT.ARC_HARVEST_FILE add index FK_AHR_ARC_HARVEST_RESULT_ID (AHF_
 alter table DB_WCT.ARC_HARVEST_RESOURCE add index FK6D84FEB12FF8F14B (AHRC_HARVEST_RESOURCE_OID), add constraint FK6D84FEB12FF8F14B foreign key (AHRC_HARVEST_RESOURCE_OID) references DB_WCT.HARVEST_RESOURCE (HRC_OID);
 alter table DB_WCT.ARC_HARVEST_RESULT add index FKE39C5380C88A38D9 (AHRS_HARVEST_RESULT_OID), add constraint FKE39C5380C88A38D9 foreign key (AHRS_HARVEST_RESULT_OID) references DB_WCT.HARVEST_RESULT (HR_OID);
 alter table DB_WCT.BANDWIDTH_RESTRICTIONS add constraint CHK_DAY check (br_day IN ('MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', 'SATURDAY', 'SUNDAY'));
-alter table DB_WCT.BANDWIDTH_RESTRICTIONS add constraint CHK_END_TIME check (br_end_time >= TO_DATE('1972-11-09 00:00:00', 'YYYY-MM-DD HH24:MI:SS') AND br_end_time <= TO_DATE('1972-11-09 23:59:59', 'YYYY-MM-DD HH24:MI:SS'));
-alter table DB_WCT.BANDWIDTH_RESTRICTIONS add constraint CHK_START_TIME check (br_start_time >= TO_DATE('1972-11-09 00:00:00', 'YYYY-MM-DD HH24:MI:SS') AND br_start_time <= TO_DATE('1972-11-09 23:59:59', 'YYYY-MM-DD HH24:MI:SS'));
+alter table DB_WCT.BANDWIDTH_RESTRICTIONS add constraint CHK_END_TIME check (br_end_time >= STR_TO_DATE('1972-11-09 00:00:00', '%Y-%m-%d %H:%i:%s') AND br_end_time <= STR_TO_DATE('1972-11-09 23:59:59', '%Y-%m-%d %H:%i:%s'));
+alter table DB_WCT.BANDWIDTH_RESTRICTIONS add constraint CHK_START_TIME check (br_start_time >= STR_TO_DATE('1972-11-09 00:00:00', '%Y-%m-%d %H:%i:%s') AND br_start_time <= STR_TO_DATE('1972-11-09 23:59:59', '%Y-%m-%d %H:%i:%s'));
 alter table DB_WCT.GROUP_MEMBER add index FK_GM_PARENT_ID (GM_PARENT_ID), add constraint FK_GM_PARENT_ID foreign key (GM_PARENT_ID) references DB_WCT.TARGET_GROUP (TG_AT_OID);
 alter table DB_WCT.GROUP_MEMBER add index FK_GM_CHILD_ID (GM_CHILD_ID), add constraint FK_GM_CHILD_ID foreign key (GM_CHILD_ID) references DB_WCT.ABSTRACT_TARGET (AT_OID);
 alter table DB_WCT.HARVEST_RESOURCE add index FK5BA2B04431A1C148 (HRC_HARVEST_RESULT_OID), add constraint FK5BA2B04431A1C148 foreign key (HRC_HARVEST_RESULT_OID) references DB_WCT.HARVEST_RESULT (HR_OID);

wct-core/db/latest/upgrade/wct-upgrade-1_6_1-to-2_0-mysql.sql

@@ -4,13 +4,10 @@ alter table DB_WCT.PROFILE add column P_HARVESTER_TYPE varchar(40) not null;
 update DB_WCT.PROFILE set P_HARVESTER_TYPE='HERITRIX1';
 
 alter table DB_WCT.PROFILE add column P_DATA_LIMIT_UNIT varchar(40);
-update DB_WCT.PROFILE set P_DATA_LIMIT_UNIT='B' where P_HARVESTER_TYPE='HERITRIX3';
 
 alter table DB_WCT.PROFILE add column P_MAX_FILE_SIZE_UNIT varchar(40);
-update DB_WCT.PROFILE set P_MAX_FILE_SIZE_UNIT='B' where P_HARVESTER_TYPE='HERITRIX3';
 
 alter table DB_WCT.PROFILE add column P_TIME_LIMIT_UNIT varchar(40);
-update DB_WCT.PROFILE set P_TIME_LIMIT_UNIT='SECOND' where P_HARVESTER_TYPE='HERITRIX3';
 
 -- PROFILE_OVERRIDES
 
@@ -21,8 +18,33 @@ add column PO_H3_OR_DATA_LIMIT bit, add column PO_H3_OR_TIME_LIMIT bit, add colu
 add column PO_H3_OR_MAX_TRANS_HOPS bit, add column PO_H3_OR_IGNORE_ROBOTS bit, add column PO_H3_OR_IGNORE_COOKIES bit, add column PO_H3_OR_BLOCK_URL bit,
 add column PO_H3_OR_INCL_URL bit;
 
-update DB_WCT.PROFILE_OVERRIDES set PO_H3_IGNORE_COOKIES = 0, PO_H3_OR_DOC_LIMIT = 0, PO_H3_OR_DATA_LIMIT = 0, PO_H3_OR_TIME_LIMIT = 0, PO_H3_OR_MAX_PATH_DEPTH = 0,
-PO_H3_OR_MAX_HOPS = 0, PO_H3_OR_MAX_TRANS_HOPS = 0, PO_H3_IGNORE_ROBOTS = 0, PO_H3_OR_IGNORE_ROBOTS = 0, PO_H3_OR_IGNORE_COOKIES = 0, PO_H3_OR_BLOCK_URL = 0, PO_H3_OR_INCL_URL = 0;
+
+-- Provide sensible defaults for new override attributes
+update DB_WCT.PROFILE_OVERRIDES set
+PO_H3_DATA_LIMIT_UNIT = 'B',
+PO_H3_EXTRACT_JS = 1,
+PO_H3_IGNORE_COOKIES = 0,
+PO_H3_OR_EXTRACT_JS = 0,
+PO_H3_OR_IGNORE_COOKIES = 0,
+PO_H3_OR_MAX_TRANS_HOPS = 0,
+PO_H3_OR_RAW_PROFILE = 0,
+PO_H3_TIME_LIMIT_UNIT = 'SECOND',
+PO_H3_DATA_LIMIT = PO_MAX_BYES,
+PO_H3_DOC_LIMIT = PO_MAX_DOCS,
+PO_H3_MAX_HOPS = PO_MAX_HOPS,
+PO_H3_MAX_PATH_DEPTH = PO_MAX_PATH_DEPTH,
+PO_H3_TIME_LIMIT = PO_MAX_TIME_SEC,
+PO_H3_IGNORE_ROBOTS = case PO_ROBOTS_POLICY when 'ignore' then 1 else 0 end,
+PO_H3_OR_BLOCK_URL = PO_OR_EXCLUSION_URI,
+PO_H3_OR_INCL_URL = PO_OR_INCLUSION_URI,
+PO_H3_OR_DATA_LIMIT = PO_OR_MAX_BYTES,
+PO_H3_OR_DOC_LIMIT = PO_OR_MAX_DOCS,
+PO_H3_OR_MAX_HOPS = PO_OR_MAX_HOPS,
+PO_H3_OR_MAX_PATH_DEPTH = PO_OR_MAX_PATH_DEPTH,
+PO_H3_OR_TIME_LIMIT = PO_OR_MAX_TIME_SEC,
+PO_H3_OR_IGNORE_ROBOTS = PO_OR_ROBOTS_POLICY
+;
+
 
 create table DB_WCT.PO_H3_BLOCK_URL (PBU_PROF_OVER_OID bigint not null, PBU_FILTER varchar(255), PBU_IX integer not null, primary key (PBU_PROF_OVER_OID, PBU_IX));
 create table DB_WCT.PO_H3_INCLUDE_URL (PIU_PROF_OVER_OID bigint not null, PIU_FILTER varchar(255), PIU_IX integer not null, primary key (PIU_PROF_OVER_OID, PIU_IX));
@@ -34,6 +56,12 @@ alter table DB_WCT.PO_H3_INCLUDE_URL add index PIU_FK_1 (PIU_PROF_OVER_OID), add
 GRANT SELECT, INSERT, UPDATE, DELETE ON DB_WCT.PO_H3_BLOCK_URL TO usr_wct@localhost;
 GRANT SELECT, INSERT, UPDATE, DELETE ON DB_WCT.PO_H3_INCLUDE_URL TO usr_wct@localhost;
 
+INSERT INTO DB_WCT.PO_H3_BLOCK_URL (PBU_PROF_OVER_OID, PBU_FILTER, PBU_IX)
+SELECT PEU_PROF_OVER_OID, PEU_FILTER, PEU_IX FROM DB_WCT.PO_EXCLUSION_URI;
+
+INSERT INTO DB_WCT.PO_H3_INCLUDE_URL (PIU_PROF_OVER_OID, PIU_FILTER, PIU_IX)
+SELECT PEU_PROF_OVER_OID, PEU_FILTER, PEU_IX FROM DB_WCT.PO_INCLUSION_URI;
+
 alter table DB_WCT.PROFILE add column P_IMPORTED bit not null default 0;
 
 alter table DB_WCT.PROFILE_OVERRIDES add column PO_H3_RAW_PROFILE text;

wct-core/db/latest/upgrade/wct-upgrade-1_6_1-to-2_0-oracle.sql

@@ -5,13 +5,10 @@ update DB_WCT.PROFILE set P_HARVESTER_TYPE='HERITRIX1';
 alter table DB_WCT.PROFILE modify P_HARVESTER_TYPE varchar2(40) not null;
 
 alter table DB_WCT.PROFILE add (P_DATA_LIMIT_UNIT varchar2(40));
-update DB_WCT.PROFILE set P_DATA_LIMIT_UNIT='B' where P_HARVESTER_TYPE='HERITRIX3';
 
 alter table DB_WCT.PROFILE add (P_MAX_FILE_SIZE_UNIT varchar2(40));
-update DB_WCT.PROFILE set P_MAX_FILE_SIZE_UNIT='B' where P_HARVESTER_TYPE='HERITRIX3';
 
 alter table DB_WCT.PROFILE add (P_TIME_LIMIT_UNIT varchar2(40));
-update DB_WCT.PROFILE set P_TIME_LIMIT_UNIT='SECOND' where P_HARVESTER_TYPE='HERITRIX3';
 
 -- PROFILE_OVERRIDES
 
@@ -22,8 +19,33 @@ PO_H3_OR_DATA_LIMIT number(1,0), PO_H3_OR_TIME_LIMIT number(1,0), PO_H3_OR_MAX_P
 PO_H3_OR_MAX_TRANS_HOPS number(1,0), PO_H3_OR_IGNORE_ROBOTS number(1,0), PO_H3_OR_IGNORE_COOKIES number(1,0), PO_H3_OR_BLOCK_URL number(1,0),
 PO_H3_OR_INCL_URL number(1,0));
 
-update DB_WCT.PROFILE_OVERRIDES set PO_H3_IGNORE_COOKIES = 0, PO_H3_OR_DOC_LIMIT = 0, PO_H3_OR_DATA_LIMIT = 0, PO_H3_OR_TIME_LIMIT = 0, PO_H3_OR_MAX_PATH_DEPTH = 0,
-PO_H3_OR_MAX_HOPS = 0, PO_H3_OR_MAX_TRANS_HOPS = 0, PO_H3_IGNORE_ROBOTS = 0, PO_H3_OR_IGNORE_ROBOTS = 0, PO_H3_OR_IGNORE_COOKIES = 0, PO_H3_OR_BLOCK_URL = 0, PO_H3_OR_INCL_URL = 0;
+
+-- Provide sensible defaults for new override attributes
+update DB_WCT.PROFILE_OVERRIDES set
+PO_H3_DATA_LIMIT_UNIT = 'B',
+PO_H3_EXTRACT_JS = 1,
+PO_H3_IGNORE_COOKIES = 0,
+PO_H3_OR_EXTRACT_JS = 0,
+PO_H3_OR_IGNORE_COOKIES = 0,
+PO_H3_OR_MAX_TRANS_HOPS = 0,
+PO_H3_OR_RAW_PROFILE = 0,
+PO_H3_TIME_LIMIT_UNIT = 'SECOND',
+PO_H3_DATA_LIMIT = PO_MAX_BYES,
+PO_H3_DOC_LIMIT = PO_MAX_DOCS,
+PO_H3_MAX_HOPS = PO_MAX_HOPS,
+PO_H3_MAX_PATH_DEPTH = PO_MAX_PATH_DEPTH,
+PO_H3_TIME_LIMIT = PO_MAX_TIME_SEC,
+PO_H3_IGNORE_ROBOTS = case PO_ROBOTS_POLICY when 'ignore' then 1 else 0 end,
+PO_H3_OR_BLOCK_URL = PO_OR_EXCLUSION_URI,
+PO_H3_OR_INCL_URL = PO_OR_INCLUSION_URI,
+PO_H3_OR_DATA_LIMIT = PO_OR_MAX_BYTES,
+PO_H3_OR_DOC_LIMIT = PO_OR_MAX_DOCS,
+PO_H3_OR_MAX_HOPS = PO_OR_MAX_HOPS,
+PO_H3_OR_MAX_PATH_DEPTH = PO_OR_MAX_PATH_DEPTH,
+PO_H3_OR_TIME_LIMIT = PO_OR_MAX_TIME_SEC,
+PO_H3_OR_IGNORE_ROBOTS = PO_OR_ROBOTS_POLICY
+;
+
 
 create table DB_WCT.PO_H3_BLOCK_URL (PBU_PROF_OVER_OID number(19,0) not null, PBU_FILTER varchar2(255), PBU_IX number(10,0) not null, primary key (PBU_PROF_OVER_OID, PBU_IX));
 create table DB_WCT.PO_H3_INCLUDE_URL (PIU_PROF_OVER_OID number(19,0) not null, PIU_FILTER varchar2(255), PIU_IX number(10,0) not null, primary key (PIU_PROF_OVER_OID, PIU_IX));
@@ -34,6 +56,13 @@ alter table DB_WCT.PO_H3_INCLUDE_URL add constraint PIU_FK_1 foreign key (PIU_PR
 GRANT SELECT, INSERT, UPDATE, DELETE ON DB_WCT.PO_H3_BLOCK_URL TO USR_WCT;
 GRANT SELECT, INSERT, UPDATE, DELETE ON DB_WCT.PO_H3_INCLUDE_URL TO USR_WCT;
 
+INSERT INTO DB_WCT.PO_H3_BLOCK_URL (PBU_PROF_OVER_OID, PBU_FILTER, PBU_IX)
+SELECT PEU_PROF_OVER_OID, PEU_FILTER, PEU_IX FROM DB_WCT.PO_EXCLUSION_URI;
+
+INSERT INTO DB_WCT.PO_H3_INCLUDE_URL (PIU_PROF_OVER_OID, PIU_FILTER, PIU_IX)
+SELECT PEU_PROF_OVER_OID, PEU_FILTER, PEU_IX FROM DB_WCT.PO_INCLUSION_URI;
+
+
 alter table DB_WCT.PROFILE add (P_IMPORTED number(1,0) default 0 not null);
 
 

wct-core/db/latest/upgrade/wct-upgrade-1_6_1-to-2_0-postgres.sql

@@ -5,13 +5,10 @@ update DB_WCT.PROFILE set P_HARVESTER_TYPE='HERITRIX1';
 alter table DB_WCT.PROFILE  alter column P_HARVESTER_TYPE set not null;
 
 alter table DB_WCT.PROFILE add column P_DATA_LIMIT_UNIT varchar(40);
-update DB_WCT.PROFILE set P_DATA_LIMIT_UNIT='B' where P_HARVESTER_TYPE='HERITRIX3';
 
 alter table DB_WCT.PROFILE add column P_MAX_FILE_SIZE_UNIT varchar(40);
-update DB_WCT.PROFILE set P_MAX_FILE_SIZE_UNIT='B' where P_HARVESTER_TYPE='HERITRIX3';
 
 alter table DB_WCT.PROFILE add column P_TIME_LIMIT_UNIT varchar(40);
-update DB_WCT.PROFILE set P_TIME_LIMIT_UNIT='SECOND' where P_HARVESTER_TYPE='HERITRIX3';
 
 -- PROFILE_OVERRIDES
 
@@ -22,8 +19,33 @@ add column PO_H3_OR_DATA_LIMIT bool, add column PO_H3_OR_TIME_LIMIT bool, add co
 add column PO_H3_OR_MAX_TRANS_HOPS bool, add column PO_H3_OR_IGNORE_ROBOTS bool, add column PO_H3_OR_IGNORE_COOKIES bool, add column PO_H3_OR_BLOCK_URL bool,
 add column PO_H3_OR_INCL_URL bool;
 
-update DB_WCT.PROFILE_OVERRIDES set PO_H3_IGNORE_COOKIES = false, PO_H3_OR_DOC_LIMIT = false, PO_H3_OR_DATA_LIMIT = false, PO_H3_OR_TIME_LIMIT = false, PO_H3_OR_MAX_PATH_DEPTH = false,
-PO_H3_OR_MAX_HOPS = false, PO_H3_OR_MAX_TRANS_HOPS = false, PO_H3_IGNORE_ROBOTS = false, PO_H3_OR_IGNORE_ROBOTS = false, PO_H3_OR_IGNORE_COOKIES = false, PO_H3_OR_BLOCK_URL = false, PO_H3_OR_INCL_URL = false;
+
+-- Provide sensible defaults for new override attributes
+update DB_WCT.PROFILE_OVERRIDES set
+PO_H3_DATA_LIMIT_UNIT = 'B',
+PO_H3_EXTRACT_JS = true,
+PO_H3_IGNORE_COOKIES = false,
+PO_H3_OR_EXTRACT_JS = false,
+PO_H3_OR_IGNORE_COOKIES = false,
+PO_H3_OR_MAX_TRANS_HOPS = false,
+PO_H3_OR_RAW_PROFILE = false,
+PO_H3_TIME_LIMIT_UNIT = 'SECOND',
+PO_H3_DATA_LIMIT = PO_MAX_BYES,
+PO_H3_DOC_LIMIT = PO_MAX_DOCS,
+PO_H3_MAX_HOPS = PO_MAX_HOPS,
+PO_H3_MAX_PATH_DEPTH = PO_MAX_PATH_DEPTH,
+PO_H3_TIME_LIMIT = PO_MAX_TIME_SEC,
+PO_H3_IGNORE_ROBOTS = case PO_ROBOTS_POLICY when 'ignore' then true else false end,
+PO_H3_OR_BLOCK_URL = PO_OR_EXCLUSION_URI,
+PO_H3_OR_INCL_URL = PO_OR_INCLUSION_URI,
+PO_H3_OR_DATA_LIMIT = PO_OR_MAX_BYTES,
+PO_H3_OR_DOC_LIMIT = PO_OR_MAX_DOCS,
+PO_H3_OR_MAX_HOPS = PO_OR_MAX_HOPS,
+PO_H3_OR_MAX_PATH_DEPTH = PO_OR_MAX_PATH_DEPTH,
+PO_H3_OR_TIME_LIMIT = PO_OR_MAX_TIME_SEC,
+PO_H3_OR_IGNORE_ROBOTS = PO_OR_ROBOTS_POLICY
+;
+
 
 create table DB_WCT.PO_H3_BLOCK_URL (PBU_PROF_OVER_OID int8 not null, PBU_FILTER varchar(255), PBU_IX int4 not null, primary key (PBU_PROF_OVER_OID, PBU_IX));
 create table DB_WCT.PO_H3_INCLUDE_URL (PIU_PROF_OVER_OID int8 not null, PIU_FILTER varchar(255), PIU_IX int4 not null, primary key (PIU_PROF_OVER_OID, PIU_IX));
@@ -34,6 +56,13 @@ alter table DB_WCT.PO_H3_INCLUDE_URL add constraint PIU_FK_1 foreign key (PIU_PR
 GRANT SELECT, INSERT, UPDATE, DELETE ON DB_WCT.PO_H3_BLOCK_URL TO USR_WCT;
 GRANT SELECT, INSERT, UPDATE, DELETE ON DB_WCT.PO_H3_INCLUDE_URL TO USR_WCT;
 
+INSERT INTO DB_WCT.PO_H3_BLOCK_URL (PBU_PROF_OVER_OID, PBU_FILTER, PBU_IX)
+SELECT PEU_PROF_OVER_OID, PEU_FILTER, PEU_IX FROM DB_WCT.PO_EXCLUSION_URI;
+
+INSERT INTO DB_WCT.PO_H3_INCLUDE_URL (PIU_PROF_OVER_OID, PIU_FILTER, PIU_IX)
+SELECT PEU_PROF_OVER_OID, PEU_FILTER, PEU_IX FROM DB_WCT.PO_INCLUSION_URI;
+
+
 alter table DB_WCT.PROFILE add column P_IMPORTED bool not null default false;
 
 alter table DB_WCT.PROFILE_OVERRIDES add column PO_H3_RAW_PROFILE text;