From 6626c009ac19be2fdf5815b70eabbdfdf2b0e434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 30 Jan 2019 13:33:23 +0100 Subject: [PATCH 01/20] ARROW-4414: [C++] Stop using cmake COMMAND_EXPAND_LISTS because it breaks package builds for older distros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit COMMAND_EXPAND_LISTS option of add_custom_command is too new on Ubuntu Xenial and Debian stretch. It's available since CMake 3.8 Linux package builds: [kszucs/crossbow/build-434](https://github.com/kszucs/crossbow/branches/all?utf8=%E2%9C%93&query=434) (the xenial crossbow error is unrelated) Author: Krisztián Szűcs Closes #3522 from kszucs/ARROW-4414 and squashes the following commits: 23dde8a3 revert removing cmake from conda_env_cpp ca52b309 use system cmake on travis and in docker images; remove cmake_expand_lists --- ci/travis_script_python.sh | 1 - cpp/src/gandiva/precompiled/CMakeLists.txt | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 60335d966846b..27d75da74893e 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -48,7 +48,6 @@ fi conda create -y -q -p $CONDA_ENV_DIR \ --file $TRAVIS_BUILD_DIR/ci/conda_env_python.yml \ nomkl \ - cmake \ pip \ numpy=1.14 \ python=${PYTHON_VERSION} \ diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index 6e0a0926d3155..2b629c7a9e4f2 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -44,8 +44,7 @@ foreach(SRC_FILE ${PRECOMPILED_SRCS}) -O3 -c ${ABSOLUTE_SRC} -o ${BC_FILE} ${ARROW_GANDIVA_PC_CXX_FLAGS} -I${CMAKE_SOURCE_DIR}/src - DEPENDS ${SRC_FILE} - COMMAND_EXPAND_LISTS) + DEPENDS ${SRC_FILE}) list(APPEND BC_FILES ${BC_FILE}) endforeach() From 8e195327149b670de2cd7a8cfe75bbd6f71c6b49 Mon Sep 17 00:00:00 2001 From: Mike Pigott Date: Wed, 30 Jan 2019 08:01:42 -0600 Subject: [PATCH 02/20] ARROW-3965 [Java] JDBC-To-Arrow Configuration https://issues.apache.org/jira/browse/ARROW-3965 This creates an object which configures the BaseAllocator and Calendar used during to configure the translation from a JDBC ResultSet to an Arrow vector. Author: Mike Pigott Author: Michael Pigott Closes #3133 from mikepigott/jdbc-to-arrow-config and squashes the following commits: be9542686 ARROW-3965: JDBC-To-Arrow Config Builder javadocs. d6c64a775 ARROW-3965: JdbcToArrowConfigBuilder d7ca982f9 Merge branch 'master' into jdbc-to-arrow-config 789c8c84d Merge pull request #4 from apache/master e5b19eeeb Merge pull request #3 from apache/master 3b17c2976 Merge pull request #2 from apache/master 5b1b364dd Merge branch 'master' into jdbc-to-arrow-config 881c6c83e Merge pull request #1 from apache/master bb3165b93 Updating the function calls to use the JdbcToArrowConfig versions. 68c91e7a7 Modifying the jdbcToArrowSchema and jdbcToArrowVectors methods to receive JdbcToArrowConfig objects. 8d6cf0089 Documentation for public static VectorSchemaRoot sqlToArrow(Connection connection, String query, JdbcToArrowConfig config) 4f1260ce9 Adding documentation for public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, JdbcToArrowConfig config) df632e36d Updating the SQL tests to include JdbcToArrowConfig versions. b2700448d Updated validaton & documentation, and unit tests for the new JdbcToArrowConfig. da77cbe8c Creating a configuration class for the JDBC-to-Arrow converter. --- .../arrow/adapter/jdbc/JdbcToArrow.java | 56 ++++++++-- .../arrow/adapter/jdbc/JdbcToArrowConfig.java | 73 +++++++++++++ .../jdbc/JdbcToArrowConfigBuilder.java | 103 ++++++++++++++++++ .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 58 ++++++++-- .../adapter/jdbc/AbstractJdbcToArrowTest.java | 4 +- .../adapter/jdbc/JdbcToArrowConfigTest.java | 94 ++++++++++++++++ .../jdbc/h2/JdbcToArrowCharSetTest.java | 8 ++ .../jdbc/h2/JdbcToArrowDataTypesTest.java | 8 ++ .../adapter/jdbc/h2/JdbcToArrowNullTest.java | 8 ++ .../adapter/jdbc/h2/JdbcToArrowTest.java | 8 ++ .../jdbc/h2/JdbcToArrowTimeZoneTest.java | 12 ++ 11 files changed, 410 insertions(+), 22 deletions(-) create mode 100644 java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java create mode 100644 java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 14e4368368dda..fd320367f77b6 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -89,7 +89,9 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, B Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); - return sqlToArrow(connection, query, allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); + JdbcToArrowConfig config = + new JdbcToArrowConfig(allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); + return sqlToArrow(connection, query, config); } /** @@ -115,8 +117,29 @@ public static VectorSchemaRoot sqlToArrow( Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + return sqlToArrow(connection, query, new JdbcToArrowConfig(allocator, calendar)); + } + + /** + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * + * @param connection Database connection to be used. This method will not close the passed connection object. + * Since the caller has passed the connection object it's the responsibility of the caller + * to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @param config Configuration + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as + * ResultSet and Statement objects. + */ + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, JdbcToArrowConfig config) + throws SQLException, IOException { + Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); + Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); + Preconditions.checkNotNull(config, "The configuration cannot be null"); + try (Statement stmt = connection.createStatement()) { - return sqlToArrow(stmt.executeQuery(query), allocator, calendar); + return sqlToArrow(stmt.executeQuery(query), config); } } @@ -147,7 +170,9 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator all Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); - return sqlToArrow(resultSet, allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); + JdbcToArrowConfig config = + new JdbcToArrowConfig(allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); + return sqlToArrow(resultSet, config); } /** @@ -162,10 +187,7 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); Preconditions.checkNotNull(calendar, "Calendar object can not be null"); - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - VectorSchemaRoot root = sqlToArrow(resultSet, rootAllocator, calendar); - - return root; + return sqlToArrow(resultSet, new JdbcToArrowConfig(new RootAllocator(Integer.MAX_VALUE), calendar)); } /** @@ -183,9 +205,25 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator all Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + return sqlToArrow(resultSet, new JdbcToArrowConfig(allocator, calendar)); + } + + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * + * @param resultSet ResultSet to use to fetch the data from underlying database + * @param config Configuration of the conversion from JDBC to Arrow. + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException on error + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, JdbcToArrowConfig config) + throws SQLException, IOException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + Preconditions.checkNotNull(config, "The configuration cannot be null"); + VectorSchemaRoot root = VectorSchemaRoot.create( - JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), calendar), allocator); - JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root, calendar); + JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), config), config.getAllocator()); + JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root, config); return root; } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java new file mode 100644 index 0000000000000..59813a830cbed --- /dev/null +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import java.util.Calendar; + +import org.apache.arrow.memory.BaseAllocator; + +import com.google.common.base.Preconditions; + +/** + * This class configures the JDBC-to-Arrow conversion process. + *

+ * The allocator is used to construct the {@link org.apache.arrow.vector.VectorSchemaRoot}, + * and the calendar is used to define the time zone of any {@link org.apahe.arrow.vector.pojo.ArrowType.Timestamp} + * fields that are created during the conversion. + *

+ *

+ * Neither field may be null. + *

+ */ +public final class JdbcToArrowConfig { + private Calendar calendar; + private BaseAllocator allocator; + + /** + * Constructs a new configuration from the provided allocator and calendar. The allocator + * is used when constructing the Arrow vectors from the ResultSet, and the calendar is used to define + * Arrow Timestamp fields, and to read time-based fields from the JDBC ResultSet. + * + * @param allocator The memory allocator to construct the Arrow vectors with. + * @param calendar The calendar to use when constructing Timestamp fields and reading time-based results. + */ + JdbcToArrowConfig(BaseAllocator allocator, Calendar calendar) { + Preconditions.checkNotNull(allocator, "Memory allocator cannot be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + + this.allocator = allocator; + this.calendar = calendar; + } + + /** + * The calendar to use when defining Arrow Timestamp fields + * and retrieving time-based fields from the database. + * @return the calendar. + */ + public Calendar getCalendar() { + return calendar; + } + + /** + * The Arrow memory allocator. + * @return the allocator. + */ + public BaseAllocator getAllocator() { + return allocator; + } +} diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java new file mode 100644 index 0000000000000..df97c3a975196 --- /dev/null +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import java.util.Calendar; + +import org.apache.arrow.memory.BaseAllocator; + +import com.google.common.base.Preconditions; + +/** + * This class builds {@link JdbcToArrowConfig}s. + */ +public class JdbcToArrowConfigBuilder { + private Calendar calendar; + private BaseAllocator allocator; + + /** + * Default constructor for the JdbcToArrowConfigBuilder}. + * Use the setter methods for the allocator and calendar; both must be + * set. Otherwise, {@link #build()} will throw a {@link NullPointerException}. + */ + public JdbcToArrowConfigBuilder() { + this.allocator = null; + this.calendar = null; + } + + /** + * Constructor for the JdbcToArrowConfigBuilder. Both the + * allocator and calendar are required. A {@link NullPointerException} + * will be thrown if one of the arguments is null. + *

+ * The allocator is used to construct Arrow vectors from the JDBC ResultSet. + * The calendar is used to determine the time zone of {@link java.sql.Timestamp} + * fields and convert {@link java.sql.Date}, {@link java.sql.Time}, and + * {@link java.sql.Timestamp} fields to a single, common time zone when reading + * from the result set. + *

+ * + * @param allocator The Arrow Vector memory allocator. + * @param calendar The calendar to use when constructing timestamp fields. + */ + public JdbcToArrowConfigBuilder(BaseAllocator allocator, Calendar calendar) { + this(); + + Preconditions.checkNotNull(allocator, "Memory allocator cannot be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + + this.allocator = allocator; + this.calendar = calendar; + } + + /** + * Sets the memory allocator to use when constructing the Arrow vectors from the ResultSet. + * + * @param allocator the allocator to set. + * @exception NullPointerException if allocator is null. + */ + public JdbcToArrowConfigBuilder setAllocator(BaseAllocator allocator) { + Preconditions.checkNotNull(allocator, "Memory allocator cannot be null"); + this.allocator = allocator; + return this; + } + + /** + * Sets the {@link Calendar} to use when constructing timestamp fields in the + * Arrow schema, and reading time-based fields from the JDBC ResultSet. + * + * @param calendar the calendar to set. + * @exception NullPointerExeption if calendar is null. + */ + public JdbcToArrowConfigBuilder setCalendar(Calendar calendar) { + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + this.calendar = calendar; + return this; + } + + /** + * This builds the {@link JdbcToArrowConfig} from the provided + * {@link BaseAllocator} and {@link Calendar}. + * + * @return The built {@link JdbcToArrowConfig} + * @throws NullPointerException if either the allocator or calendar was not set. + */ + public JdbcToArrowConfig build() { + return new JdbcToArrowConfig(allocator, calendar); + } +} diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 3425fa6471e87..d48cfe2197b0c 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -38,6 +38,7 @@ import java.util.Calendar; import java.util.List; +import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -90,6 +91,21 @@ public class JdbcToArrowUtils { private static final int DEFAULT_STREAM_BUFFER_SIZE = 1024; private static final int DEFAULT_CLOB_SUBSTRING_READ_SIZE = 256; + /** + * Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}. + * + * @param rsmd The ResultSetMetaData containing the results, to read the JDBC metadata from. + * @param calendar The calendar to use the time zone field of, to construct Timestamp fields from. + * @return {@link Schema} + * @throws SQLException on error + */ + public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar) throws SQLException { + Preconditions.checkNotNull(rsmd, "JDBC ResultSetMetaData object can't be null"); + Preconditions.checkNotNull(calendar, "Calendar object can't be null"); + + return jdbcToArrowSchema(rsmd, new JdbcToArrowConfig(new RootAllocator(0), calendar)); + } + /** * Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}. * @@ -120,14 +136,14 @@ public class JdbcToArrowUtils { * CLOB --> ArrowType.Utf8 * BLOB --> ArrowType.Binary * - * @param rsmd ResultSetMetaData + * @param rsmd The ResultSetMetaData containing the results, to read the JDBC metadata from. + * @param config The configuration to use when constructing the schema. * @return {@link Schema} * @throws SQLException on error */ - public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar) throws SQLException { - + public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, JdbcToArrowConfig config) throws SQLException { Preconditions.checkNotNull(rsmd, "JDBC ResultSetMetaData object can't be null"); - Preconditions.checkNotNull(calendar, "Calendar object can't be null"); + Preconditions.checkNotNull(config, "The configuration object must not be null"); List fields = new ArrayList<>(); int columnCount = rsmd.getColumnCount(); @@ -179,7 +195,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar break; case Types.TIMESTAMP: fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, - calendar.getTimeZone().getID())), null)); + config.getCalendar().getTimeZone().getID())), null)); break; case Types.BINARY: case Types.VARBINARY: @@ -222,17 +238,37 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { * Iterate the given JDBC {@link ResultSet} object to fetch the data and transpose it to populate * the given Arrow Vector objects. * - * @param rs ResultSet to use to fetch the data from underlying database - * @param root Arrow {@link VectorSchemaRoot} object to populate + * @param rs ResultSet to use to fetch the data from underlying database + * @param root Arrow {@link VectorSchemaRoot} object to populate + * @param calendar The calendar to use when reading time-based data. * @throws SQLException on error */ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calendar calendar) throws SQLException, IOException { Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); - Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); + Preconditions.checkNotNull(root, "Vector Schema cannot be null"); Preconditions.checkNotNull(calendar, "Calendar object can't be null"); + jdbcToArrowVectors(rs, root, new JdbcToArrowConfig(new RootAllocator(0), calendar)); + } + + /** + * Iterate the given JDBC {@link ResultSet} object to fetch the data and transpose it to populate + * the given Arrow Vector objects. + * + * @param rs ResultSet to use to fetch the data from underlying database + * @param root Arrow {@link VectorSchemaRoot} object to populate + * @param config The configuration to use when reading the data. + * @throws SQLException on error + */ + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, JdbcToArrowConfig config) + throws SQLException, IOException { + + Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); + Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); + Preconditions.checkNotNull(config, "JDBC-to-Arrow configuration cannot be null"); + ResultSetMetaData rsmd = rs.getMetaData(); int columnCount = rsmd.getColumnCount(); @@ -289,16 +325,16 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calen break; case Types.DATE: updateVector((DateMilliVector) root.getVector(columnName), - rs.getDate(i, calendar), !rs.wasNull(), rowCount); + rs.getDate(i, config.getCalendar()), !rs.wasNull(), rowCount); break; case Types.TIME: updateVector((TimeMilliVector) root.getVector(columnName), - rs.getTime(i, calendar), !rs.wasNull(), rowCount); + rs.getTime(i, config.getCalendar()), !rs.wasNull(), rowCount); break; case Types.TIMESTAMP: // TODO: Need to handle precision such as milli, micro, nano updateVector((TimeStampVector) root.getVector(columnName), - rs.getTimestamp(i, calendar), !rs.wasNull(), rowCount); + rs.getTimestamp(i, config.getCalendar()), !rs.wasNull(), rowCount); break; case Types.BINARY: case Types.VARBINARY: diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java index a147babc4524d..b1a93291d2be7 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -45,7 +45,7 @@ public abstract class AbstractJdbcToArrowTest { * @return Table object * @throws IOException on error */ - protected static Table getTable(String ymlFilePath, Class clss) throws IOException { + protected static Table getTable(String ymlFilePath, @SuppressWarnings("rawtypes") Class clss) throws IOException { return new ObjectMapper(new YAMLFactory()).readValue( clss.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); } @@ -94,7 +94,7 @@ public void destroy() throws SQLException { * @throws ClassNotFoundException on error * @throws IOException on error */ - public static Object[][] prepareTestData(String[] testFiles, Class clss) + public static Object[][] prepareTestData(String[] testFiles, @SuppressWarnings("rawtypes") Class clss) throws SQLException, ClassNotFoundException, IOException { Object[][] tableArr = new Object[testFiles.length][]; int i = 0; diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java new file mode 100644 index 0000000000000..b4f92fa417026 --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import static org.junit.Assert.*; + +import java.util.Calendar; +import java.util.Locale; +import java.util.TimeZone; + +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.junit.Test; + +public class JdbcToArrowConfigTest { + + private static final RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + private static final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT); + + @Test(expected = NullPointerException.class) + public void testConfigNullArguments() { + new JdbcToArrowConfig(null, null); + } + + @Test(expected = NullPointerException.class) + public void testBuilderNullArguments() { + new JdbcToArrowConfigBuilder(null, null); + } + + @Test(expected = NullPointerException.class) + public void testConfigNullCalendar() { + new JdbcToArrowConfig(allocator, null); + } + + @Test(expected = NullPointerException.class) + public void testBuilderNullCalendar() { + new JdbcToArrowConfigBuilder(allocator, null); + } + + @Test(expected = NullPointerException.class) + public void testConfigNullAllocator() { + new JdbcToArrowConfig(null, calendar); + } + + @Test(expected = NullPointerException.class) + public void testBuilderNullAllocator() { + new JdbcToArrowConfigBuilder(null, calendar); + } + + @Test(expected = NullPointerException.class) + public void testSetNullAllocator() { + JdbcToArrowConfigBuilder builder = new JdbcToArrowConfigBuilder(allocator, calendar); + builder.setAllocator(null); + } + + @Test(expected = NullPointerException.class) + public void testSetNullCalendar() { + JdbcToArrowConfigBuilder builder = new JdbcToArrowConfigBuilder(allocator, calendar); + builder.setCalendar(null); + } + + @Test + public void testConfig() { + JdbcToArrowConfigBuilder builder = new JdbcToArrowConfigBuilder(allocator, calendar); + JdbcToArrowConfig config = builder.build(); + + assertTrue(allocator == config.getAllocator()); + assertTrue(calendar == config.getCalendar()); + + Calendar newCalendar = Calendar.getInstance(); + BaseAllocator newAllocator = new RootAllocator(Integer.SIZE); + + builder.setAllocator(newAllocator).setCalendar(newCalendar); + config = builder.build(); + + assertTrue(newAllocator == config.getAllocator()); + assertTrue(newCalendar == config.getCalendar()); + } +} diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java index c7dff431da650..d33c07a075e81 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java @@ -31,6 +31,7 @@ import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VarCharVector; @@ -116,6 +117,13 @@ public void testJdbcToArroValues() throws SQLException, IOException { new RootAllocator(Integer.MAX_VALUE))); testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build())); + testDataSets(JdbcToArrow.sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build())); } /** diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java index f6cd7645e0cac..5bdb38ff8be9f 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java @@ -40,6 +40,7 @@ import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; @@ -142,6 +143,13 @@ public void testJdbcToArroValues() throws SQLException, IOException { testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build())); + testDataSets(JdbcToArrow.sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build())); } /** diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java index 7933732f014b0..629bcfeaed304 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java @@ -27,6 +27,7 @@ import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; @@ -99,6 +100,13 @@ public void testJdbcToArroValues() throws SQLException, IOException { testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build())); + testDataSets(JdbcToArrow.sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build())); } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index 4cbfeafb0a531..f74e683d7d753 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -48,6 +48,7 @@ import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; @@ -133,6 +134,13 @@ public void testJdbcToArroValues() throws SQLException, IOException { new RootAllocator(Integer.MAX_VALUE))); testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build())); + testDataSets(JdbcToArrow.sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build())); } /** diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java index 93dc10477f697..fee56c7c07e91 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java @@ -30,6 +30,7 @@ import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.DateMilliVector; @@ -105,6 +106,17 @@ public void testJdbcToArroValues() throws SQLException, IOException { new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); + testDataSets(JdbcToArrow.sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))).build())); + testDataSets(JdbcToArrow.sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))).build())); } /** From cf7bd131949366b8df901a89e31651f227f13197 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 30 Jan 2019 15:51:05 +0100 Subject: [PATCH 03/20] ARROW-4328: Add a ARROW_USE_OLD_CXXABI configure var to R Sets GLIBCXX_USE_CXX11_ABI=0 so that the R lib will be compatible with a shared library built with -DARROW_TENSORFLOW=ON Author: Peter Closes #3409 from cHYzZQo/r-with-tensorflow and squashes the following commits: d7dc98b7 ARROW-4328: Add a ARROW_USE_OLD_CXXABI configure var to R --- r/README.md | 7 +++++++ r/configure | 8 +++++++- r/src/Makevars.in | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/r/README.md b/r/README.md index 987d0c24a185b..1e2cb2c98ced3 100644 --- a/r/README.md +++ b/r/README.md @@ -24,6 +24,13 @@ Then the R package: devtools::install_github("apache/arrow/r") ``` +If libarrow was built with the old CXXABI then you need to pass +the ARROW_USE_OLD_CXXABI configuration variable. + +``` r +devtools::install_github("apache/arrow/r", args=c("--configure-vars=ARROW_USE_OLD_CXXABI=1")) +``` + ## Example ``` r diff --git a/r/configure b/r/configure index c17fd4c2ef624..9df1f9d048553 100755 --- a/r/configure +++ b/r/configure @@ -71,6 +71,12 @@ CXX11FLAGS=$("${R_HOME}"/bin/R CMD config CXX11FLAGS) CXX11STD=$("${R_HOME}"/bin/R CMD config CXX11STD) CPPFLAGS=$("${R_HOME}"/bin/R CMD config CPPFLAGS) +# If libarrow uses the old GLIBCXX ABI, so we have to use it too +PKG_CXXFLAGS="$C_VISIBILITY" +if [ "$ARROW_USE_OLD_CXXABI" ]; then + PKG_CXXFLAGS="$PKG_CXXFLAGS -D_GLIBCXX_USE_CXX11_ABI=0" +fi + # Test configuration echo "#include $PKG_TEST_HEADER" | ${CXXCPP} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX11FLAGS} ${CXX11STD} -xc++ - >/dev/null 2>&1 @@ -91,7 +97,7 @@ if [ $? -ne 0 ]; then fi # Write to Makevars -sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" -e "s|@visibility@|$C_VISIBILITY|" src/Makevars.in > src/Makevars +sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" -e "s|@pkgcxxflags@|$PKG_CXXFLAGS|" src/Makevars.in > src/Makevars # Success exit 0 diff --git a/r/src/Makevars.in b/r/src/Makevars.in index a0d5fed10bab8..0d2808736c06c 100644 --- a/r/src/Makevars.in +++ b/r/src/Makevars.in @@ -16,7 +16,7 @@ # under the License. PKG_CPPFLAGS=@cflags@ -PKG_CXXFLAGS=@visibility@ +PKG_CXXFLAGS=@pkgcxxflags@ CXX_STD=CXX11 PKG_LIBS=@libs@ -Wl,-rpath,/usr/local/lib #CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" From 27f60ba2f6cff9372261f4cedd721b0f982808fd Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 30 Jan 2019 09:44:54 -0600 Subject: [PATCH 04/20] PARQUET-1519: [C++] Hide TypedColumnReader implementation behind virtual interfaces, remove use of "extern template class" This leaves the public API for reading columns unchanged. This avoids linking brittleness with "extern template class" and makes the ABI more stable, for whatever that's worth If this is deemed uncontroversial (and I hope that it is) I'll do the same for the column writers in a separate patch Author: Wes McKinney Closes #3525 from wesm/PARQUET-1519 and squashes the following commits: 632ccded8 Fix benchmarks, clang warnings f078769c3 cruft fd99f499b Hide TypedColumnReader implementation behind virtual interfaces, remove use of extern template class --- cpp/build-support/lintutils.py | 5 +- cpp/src/parquet/column-io-benchmark.cc | 7 +- cpp/src/parquet/column_reader.cc | 354 +++++++++++++++++++--- cpp/src/parquet/column_reader.h | 389 ++++--------------------- cpp/src/parquet/column_writer-test.cc | 5 +- cpp/src/parquet/encoding.cc | 3 +- 6 files changed, 375 insertions(+), 388 deletions(-) diff --git a/cpp/build-support/lintutils.py b/cpp/build-support/lintutils.py index 0a54daa1ee9bf..012d42bd696a2 100644 --- a/cpp/build-support/lintutils.py +++ b/cpp/build-support/lintutils.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import multiprocessing as mp import os from fnmatch import fnmatch from subprocess import Popen @@ -49,12 +50,12 @@ def run_parallel(cmds, **kwargs): """ Run each of cmds (with shared **kwargs) using subprocess.Popen then wait for all of them to complete. - Runs batches of os.cpu_count() * 2 from cmds + Runs batches of multiprocessing.cpu_count() * 2 from cmds returns a list of tuples containing each process' returncode, stdout, stderr """ complete = [] - for cmds_batch in chunk(cmds, os.cpu_count() * 2): + for cmds_batch in chunk(cmds, mp.cpu_count() * 2): procs_batch = [Popen(cmd, **kwargs) for cmd in cmds_batch] for proc in procs_batch: stdout, stderr = proc.communicate() diff --git a/cpp/src/parquet/column-io-benchmark.cc b/cpp/src/parquet/column-io-benchmark.cc index 3e32b2a837815..c648d562649d1 100644 --- a/cpp/src/parquet/column-io-benchmark.cc +++ b/cpp/src/parquet/column-io-benchmark.cc @@ -108,12 +108,13 @@ BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::OPTIONAL, Compression::ZSTD) BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::REPEATED, Compression::ZSTD) ->Range(1024, 65536); -std::unique_ptr BuildReader(std::shared_ptr& buffer, +std::shared_ptr BuildReader(std::shared_ptr& buffer, int64_t num_values, ColumnDescriptor* schema) { std::unique_ptr source(new InMemoryInputStream(buffer)); std::unique_ptr page_reader = PageReader::Open(std::move(source), num_values, Compression::UNCOMPRESSED); - return std::unique_ptr(new Int64Reader(schema, std::move(page_reader))); + return std::static_pointer_cast( + ColumnReader::Make(schema, std::move(page_reader))); } template definition_levels_out(state.range(1)); std::vector repetition_levels_out(state.range(1)); while (state.KeepRunning()) { - std::unique_ptr reader = BuildReader(src, state.range(1), schema.get()); + std::shared_ptr reader = BuildReader(src, state.range(1), schema.get()); int64_t values_read = 0; for (size_t i = 0; i < values.size(); i += values_read) { reader->ReadBatch(values_out.size(), definition_levels_out.data(), diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc index 113d50a40aada..33d2f5cefb5f0 100644 --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc @@ -266,19 +266,300 @@ std::unique_ptr PageReader::Open(std::unique_ptr stream } // ---------------------------------------------------------------------- +// TypedColumnReader implementations -ColumnReader::ColumnReader(const ColumnDescriptor* descr, - std::unique_ptr pager, MemoryPool* pool) - : descr_(descr), - pager_(std::move(pager)), - num_buffered_values_(0), - num_decoded_values_(0), - pool_(pool) {} +template +class TypedColumnReaderImpl : public TypedColumnReader { + public: + using T = typename DType::c_type; + + TypedColumnReaderImpl(const ColumnDescriptor* descr, std::unique_ptr pager, + ::arrow::MemoryPool* pool) + : descr_(descr), + pager_(std::move(pager)), + num_buffered_values_(0), + num_decoded_values_(0), + pool_(pool), + current_decoder_(NULLPTR) {} + + int64_t ReadBatch(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, + T* values, int64_t* values_read) override; + + int64_t ReadBatchSpaced(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, + T* values, uint8_t* valid_bits, int64_t valid_bits_offset, + int64_t* levels_read, int64_t* values_read, + int64_t* null_count) override; + + int64_t Skip(int64_t num_rows_to_skip) override; + + bool HasNext() override { + // Either there is no data page available yet, or the data page has been + // exhausted + if (num_buffered_values_ == 0 || num_decoded_values_ == num_buffered_values_) { + if (!ReadNewPage() || num_buffered_values_ == 0) { + return false; + } + } + return true; + } + + Type::type type() const override { return descr_->physical_type(); } + + const ColumnDescriptor* descr() const override { return descr_; } + + protected: + using DecoderType = TypedDecoder; + + // Advance to the next data page + bool ReadNewPage(); + + // Read multiple definition levels into preallocated memory + // + // Returns the number of decoded definition levels + int64_t ReadDefinitionLevels(int64_t batch_size, int16_t* levels) { + if (descr_->max_definition_level() == 0) { + return 0; + } + return definition_level_decoder_.Decode(static_cast(batch_size), levels); + } + + // Read multiple repetition levels into preallocated memory + // Returns the number of decoded repetition levels + int64_t ReadRepetitionLevels(int64_t batch_size, int16_t* levels) { + if (descr_->max_repetition_level() == 0) { + return 0; + } + return repetition_level_decoder_.Decode(static_cast(batch_size), levels); + } + + int64_t available_values_current_page() const { + return num_buffered_values_ - num_decoded_values_; + } + + void ConsumeBufferedValues(int64_t num_values) { num_decoded_values_ += num_values; } + + const ColumnDescriptor* descr_; + + std::unique_ptr pager_; + std::shared_ptr current_page_; + + // Not set if full schema for this field has no optional or repeated elements + LevelDecoder definition_level_decoder_; + + // Not set for flat schemas. + LevelDecoder repetition_level_decoder_; + + // The total number of values stored in the data page. This is the maximum of + // the number of encoded definition levels or encoded values. For + // non-repeated, required columns, this is equal to the number of encoded + // values. For repeated or optional values, there may be fewer data values + // than levels, and this tells you how many encoded levels there are in that + // case. + int64_t num_buffered_values_; + + // The number of values from the current data page that have been decoded + // into memory + int64_t num_decoded_values_; + + ::arrow::MemoryPool* pool_; + + // Read up to batch_size values from the current data page into the + // pre-allocated memory T* + // + // @returns: the number of values read into the out buffer + int64_t ReadValues(int64_t batch_size, T* out); + + // Read up to batch_size values from the current data page into the + // pre-allocated memory T*, leaving spaces for null entries according + // to the def_levels. + // + // @returns: the number of values read into the out buffer + int64_t ReadValuesSpaced(int64_t batch_size, T* out, int64_t null_count, + uint8_t* valid_bits, int64_t valid_bits_offset); + + // Map of encoding type to the respective decoder object. For example, a + // column chunk's data pages may include both dictionary-encoded and + // plain-encoded data. + std::unordered_map> decoders_; + + void ConfigureDictionary(const DictionaryPage* page); + DecoderType* current_decoder_; +}; -ColumnReader::~ColumnReader() {} +template +int64_t TypedColumnReaderImpl::ReadValues(int64_t batch_size, T* out) { + int64_t num_decoded = current_decoder_->Decode(out, static_cast(batch_size)); + return num_decoded; +} template -void TypedColumnReader::ConfigureDictionary(const DictionaryPage* page) { +int64_t TypedColumnReaderImpl::ReadValuesSpaced(int64_t batch_size, T* out, + int64_t null_count, + uint8_t* valid_bits, + int64_t valid_bits_offset) { + return current_decoder_->DecodeSpaced(out, static_cast(batch_size), + static_cast(null_count), valid_bits, + valid_bits_offset); +} + +template +int64_t TypedColumnReaderImpl::ReadBatch(int64_t batch_size, int16_t* def_levels, + int16_t* rep_levels, T* values, + int64_t* values_read) { + // HasNext invokes ReadNewPage + if (!HasNext()) { + *values_read = 0; + return 0; + } + + // TODO(wesm): keep reading data pages until batch_size is reached, or the + // row group is finished + batch_size = std::min(batch_size, num_buffered_values_ - num_decoded_values_); + + int64_t num_def_levels = 0; + int64_t num_rep_levels = 0; + + int64_t values_to_read = 0; + + // If the field is required and non-repeated, there are no definition levels + if (descr_->max_definition_level() > 0 && def_levels) { + num_def_levels = ReadDefinitionLevels(batch_size, def_levels); + // TODO(wesm): this tallying of values-to-decode can be performed with better + // cache-efficiency if fused with the level decoding. + for (int64_t i = 0; i < num_def_levels; ++i) { + if (def_levels[i] == descr_->max_definition_level()) { + ++values_to_read; + } + } + } else { + // Required field, read all values + values_to_read = batch_size; + } + + // Not present for non-repeated fields + if (descr_->max_repetition_level() > 0 && rep_levels) { + num_rep_levels = ReadRepetitionLevels(batch_size, rep_levels); + if (def_levels && num_def_levels != num_rep_levels) { + throw ParquetException("Number of decoded rep / def levels did not match"); + } + } + + *values_read = ReadValues(values_to_read, values); + int64_t total_values = std::max(num_def_levels, *values_read); + ConsumeBufferedValues(total_values); + + return total_values; +} + +template +int64_t TypedColumnReaderImpl::ReadBatchSpaced( + int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, T* values, + uint8_t* valid_bits, int64_t valid_bits_offset, int64_t* levels_read, + int64_t* values_read, int64_t* null_count_out) { + // HasNext invokes ReadNewPage + if (!HasNext()) { + *levels_read = 0; + *values_read = 0; + *null_count_out = 0; + return 0; + } + + int64_t total_values; + // TODO(wesm): keep reading data pages until batch_size is reached, or the + // row group is finished + batch_size = std::min(batch_size, num_buffered_values_ - num_decoded_values_); + + // If the field is required and non-repeated, there are no definition levels + if (descr_->max_definition_level() > 0) { + int64_t num_def_levels = ReadDefinitionLevels(batch_size, def_levels); + + // Not present for non-repeated fields + if (descr_->max_repetition_level() > 0) { + int64_t num_rep_levels = ReadRepetitionLevels(batch_size, rep_levels); + if (num_def_levels != num_rep_levels) { + throw ParquetException("Number of decoded rep / def levels did not match"); + } + } + + const bool has_spaced_values = internal::HasSpacedValues(descr_); + + int64_t null_count = 0; + if (!has_spaced_values) { + int values_to_read = 0; + for (int64_t i = 0; i < num_def_levels; ++i) { + if (def_levels[i] == descr_->max_definition_level()) { + ++values_to_read; + } + } + total_values = ReadValues(values_to_read, values); + for (int64_t i = 0; i < total_values; i++) { + ::arrow::BitUtil::SetBit(valid_bits, valid_bits_offset + i); + } + *values_read = total_values; + } else { + int16_t max_definition_level = descr_->max_definition_level(); + int16_t max_repetition_level = descr_->max_repetition_level(); + internal::DefinitionLevelsToBitmap(def_levels, num_def_levels, max_definition_level, + max_repetition_level, values_read, &null_count, + valid_bits, valid_bits_offset); + total_values = ReadValuesSpaced(*values_read, values, static_cast(null_count), + valid_bits, valid_bits_offset); + } + *levels_read = num_def_levels; + *null_count_out = null_count; + + } else { + // Required field, read all values + total_values = ReadValues(batch_size, values); + for (int64_t i = 0; i < total_values; i++) { + ::arrow::BitUtil::SetBit(valid_bits, valid_bits_offset + i); + } + *null_count_out = 0; + *levels_read = total_values; + } + + ConsumeBufferedValues(*levels_read); + return total_values; +} + +template +int64_t TypedColumnReaderImpl::Skip(int64_t num_rows_to_skip) { + int64_t rows_to_skip = num_rows_to_skip; + while (HasNext() && rows_to_skip > 0) { + // If the number of rows to skip is more than the number of undecoded values, skip the + // Page. + if (rows_to_skip > (num_buffered_values_ - num_decoded_values_)) { + rows_to_skip -= num_buffered_values_ - num_decoded_values_; + num_decoded_values_ = num_buffered_values_; + } else { + // We need to read this Page + // Jump to the right offset in the Page + int64_t batch_size = 1024; // ReadBatch with a smaller memory footprint + int64_t values_read = 0; + + std::shared_ptr vals = AllocateBuffer( + this->pool_, batch_size * type_traits::value_byte_size); + std::shared_ptr def_levels = + AllocateBuffer(this->pool_, batch_size * sizeof(int16_t)); + + std::shared_ptr rep_levels = + AllocateBuffer(this->pool_, batch_size * sizeof(int16_t)); + + do { + batch_size = std::min(batch_size, rows_to_skip); + values_read = ReadBatch(static_cast(batch_size), + reinterpret_cast(def_levels->mutable_data()), + reinterpret_cast(rep_levels->mutable_data()), + reinterpret_cast(vals->mutable_data()), &values_read); + rows_to_skip -= values_read; + } while (values_read > 0 && rows_to_skip > 0); + } + } + return num_rows_to_skip - rows_to_skip; +} + +template +void TypedColumnReaderImpl::ConfigureDictionary(const DictionaryPage* page) { int encoding = static_cast(page->encoding()); if (page->encoding() == Encoding::PLAIN_DICTIONARY || page->encoding() == Encoding::PLAIN) { @@ -317,7 +598,7 @@ static bool IsDictionaryIndexEncoding(const Encoding::type& e) { } template -bool TypedColumnReader::ReadNewPage() { +bool TypedColumnReaderImpl::ReadNewPage() { // Loop until we find the next data page. const uint8_t* buffer; @@ -415,23 +696,6 @@ bool TypedColumnReader::ReadNewPage() { return true; } -// ---------------------------------------------------------------------- -// Batch read APIs - -int64_t ColumnReader::ReadDefinitionLevels(int64_t batch_size, int16_t* levels) { - if (descr_->max_definition_level() == 0) { - return 0; - } - return definition_level_decoder_.Decode(static_cast(batch_size), levels); -} - -int64_t ColumnReader::ReadRepetitionLevels(int64_t batch_size, int16_t* levels) { - if (descr_->max_repetition_level() == 0) { - return 0; - } - return repetition_level_decoder_.Decode(static_cast(batch_size), levels); -} - // ---------------------------------------------------------------------- // Dynamic column reader constructor @@ -440,21 +704,29 @@ std::shared_ptr ColumnReader::Make(const ColumnDescriptor* descr, MemoryPool* pool) { switch (descr->physical_type()) { case Type::BOOLEAN: - return std::make_shared(descr, std::move(pager), pool); + return std::make_shared>(descr, std::move(pager), + pool); case Type::INT32: - return std::make_shared(descr, std::move(pager), pool); + return std::make_shared>(descr, std::move(pager), + pool); case Type::INT64: - return std::make_shared(descr, std::move(pager), pool); + return std::make_shared>(descr, std::move(pager), + pool); case Type::INT96: - return std::make_shared(descr, std::move(pager), pool); + return std::make_shared>(descr, std::move(pager), + pool); case Type::FLOAT: - return std::make_shared(descr, std::move(pager), pool); + return std::make_shared>(descr, std::move(pager), + pool); case Type::DOUBLE: - return std::make_shared(descr, std::move(pager), pool); + return std::make_shared>(descr, std::move(pager), + pool); case Type::BYTE_ARRAY: - return std::make_shared(descr, std::move(pager), pool); + return std::make_shared>( + descr, std::move(pager), pool); case Type::FIXED_LEN_BYTE_ARRAY: - return std::make_shared(descr, std::move(pager), pool); + return std::make_shared>(descr, std::move(pager), + pool); default: ParquetException::NYI("type reader not implemented"); } @@ -462,16 +734,4 @@ std::shared_ptr ColumnReader::Make(const ColumnDescriptor* descr, return std::shared_ptr(nullptr); } -// ---------------------------------------------------------------------- -// Instantiate templated classes - -template class PARQUET_TEMPLATE_EXPORT TypedColumnReader; -template class PARQUET_TEMPLATE_EXPORT TypedColumnReader; -template class PARQUET_TEMPLATE_EXPORT TypedColumnReader; -template class PARQUET_TEMPLATE_EXPORT TypedColumnReader; -template class PARQUET_TEMPLATE_EXPORT TypedColumnReader; -template class PARQUET_TEMPLATE_EXPORT TypedColumnReader; -template class PARQUET_TEMPLATE_EXPORT TypedColumnReader; -template class PARQUET_TEMPLATE_EXPORT TypedColumnReader; - } // namespace parquet diff --git a/cpp/src/parquet/column_reader.h b/cpp/src/parquet/column_reader.h index 19513c210d327..577107deaaa3a 100644 --- a/cpp/src/parquet/column_reader.h +++ b/cpp/src/parquet/column_reader.h @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -#ifndef PARQUET_COLUMN_READER_H -#define PARQUET_COLUMN_READER_H +#pragma once #include #include @@ -102,127 +101,26 @@ class PARQUET_EXPORT PageReader { class PARQUET_EXPORT ColumnReader { public: - ColumnReader(const ColumnDescriptor*, std::unique_ptr, - ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); - virtual ~ColumnReader(); + virtual ~ColumnReader() = default; static std::shared_ptr Make( const ColumnDescriptor* descr, std::unique_ptr pager, ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); // Returns true if there are still values in this column. - bool HasNext() { - // Either there is no data page available yet, or the data page has been - // exhausted - if (num_buffered_values_ == 0 || num_decoded_values_ == num_buffered_values_) { - if (!ReadNewPage() || num_buffered_values_ == 0) { - return false; - } - } - return true; - } - - Type::type type() const { return descr_->physical_type(); } - - const ColumnDescriptor* descr() const { return descr_; } + virtual bool HasNext() = 0; - protected: - virtual bool ReadNewPage() = 0; - - // Read multiple definition levels into preallocated memory - // - // Returns the number of decoded definition levels - int64_t ReadDefinitionLevels(int64_t batch_size, int16_t* levels); - - // Read multiple repetition levels into preallocated memory - // Returns the number of decoded repetition levels - int64_t ReadRepetitionLevels(int64_t batch_size, int16_t* levels); - - int64_t available_values_current_page() const { - return num_buffered_values_ - num_decoded_values_; - } + virtual Type::type type() const = 0; - void ConsumeBufferedValues(int64_t num_values) { num_decoded_values_ += num_values; } - - const ColumnDescriptor* descr_; - - std::unique_ptr pager_; - std::shared_ptr current_page_; - - // Not set if full schema for this field has no optional or repeated elements - LevelDecoder definition_level_decoder_; - - // Not set for flat schemas. - LevelDecoder repetition_level_decoder_; - - // The total number of values stored in the data page. This is the maximum of - // the number of encoded definition levels or encoded values. For - // non-repeated, required columns, this is equal to the number of encoded - // values. For repeated or optional values, there may be fewer data values - // than levels, and this tells you how many encoded levels there are in that - // case. - int64_t num_buffered_values_; - - // The number of values from the current data page that have been decoded - // into memory - int64_t num_decoded_values_; - - ::arrow::MemoryPool* pool_; + virtual const ColumnDescriptor* descr() const = 0; }; -namespace internal { - -static inline void DefinitionLevelsToBitmap( - const int16_t* def_levels, int64_t num_def_levels, const int16_t max_definition_level, - const int16_t max_repetition_level, int64_t* values_read, int64_t* null_count, - uint8_t* valid_bits, int64_t valid_bits_offset) { - // We assume here that valid_bits is large enough to accommodate the - // additional definition levels and the ones that have already been written - ::arrow::internal::BitmapWriter valid_bits_writer(valid_bits, valid_bits_offset, - valid_bits_offset + num_def_levels); - - // TODO(itaiin): As an interim solution we are splitting the code path here - // between repeated+flat column reads, and non-repeated+nested reads. - // Those paths need to be merged in the future - for (int i = 0; i < num_def_levels; ++i) { - if (def_levels[i] == max_definition_level) { - valid_bits_writer.Set(); - } else if (max_repetition_level > 0) { - // repetition+flat case - if (def_levels[i] == (max_definition_level - 1)) { - valid_bits_writer.Clear(); - *null_count += 1; - } else { - continue; - } - } else { - // non-repeated+nested case - if (def_levels[i] < max_definition_level) { - valid_bits_writer.Clear(); - *null_count += 1; - } else { - throw ParquetException("definition level exceeds maximum"); - } - } - - valid_bits_writer.Next(); - } - valid_bits_writer.Finish(); - *values_read = valid_bits_writer.position(); -} - -} // namespace internal - // API to read values from a single column. This is a main client facing API. template -class PARQUET_TEMPLATE_CLASS_EXPORT TypedColumnReader : public ColumnReader { +class TypedColumnReader : public ColumnReader { public: typedef typename DType::c_type T; - TypedColumnReader(const ColumnDescriptor* schema, std::unique_ptr pager, - ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) - : ColumnReader(schema, std::move(pager), pool), current_decoder_(NULLPTR) {} - // Read a batch of repetition levels, definition levels, and values from the // column. // @@ -240,8 +138,8 @@ class PARQUET_TEMPLATE_CLASS_EXPORT TypedColumnReader : public ColumnReader { // This API is the same for both V1 and V2 of the DataPage // // @returns: actual number of levels read (see values_read for number of values read) - int64_t ReadBatch(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, - T* values, int64_t* values_read); + virtual int64_t ReadBatch(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, + T* values, int64_t* values_read) = 0; /// Read a batch of repetition levels, definition levels, and values from the /// column and leave spaces for null entries on the lowest level in the values @@ -277,113 +175,59 @@ class PARQUET_TEMPLATE_CLASS_EXPORT TypedColumnReader : public ColumnReader { /// (i.e. definition_level == max_definition_level - 1) /// @param[out] null_count The number of nulls on the lowest levels. /// (i.e. (values_read - null_count) is total number of non-null entries) - int64_t ReadBatchSpaced(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, - T* values, uint8_t* valid_bits, int64_t valid_bits_offset, - int64_t* levels_read, int64_t* values_read, - int64_t* null_count); + virtual int64_t ReadBatchSpaced(int64_t batch_size, int16_t* def_levels, + int16_t* rep_levels, T* values, uint8_t* valid_bits, + int64_t valid_bits_offset, int64_t* levels_read, + int64_t* values_read, int64_t* null_count) = 0; // Skip reading levels // Returns the number of levels skipped - int64_t Skip(int64_t num_rows_to_skip); - - private: - using DecoderType = TypedDecoder; - - // Advance to the next data page - bool ReadNewPage() override; - - // Read up to batch_size values from the current data page into the - // pre-allocated memory T* - // - // @returns: the number of values read into the out buffer - int64_t ReadValues(int64_t batch_size, T* out); - - // Read up to batch_size values from the current data page into the - // pre-allocated memory T*, leaving spaces for null entries according - // to the def_levels. - // - // @returns: the number of values read into the out buffer - int64_t ReadValuesSpaced(int64_t batch_size, T* out, int64_t null_count, - uint8_t* valid_bits, int64_t valid_bits_offset); - - // Map of encoding type to the respective decoder object. For example, a - // column chunk's data pages may include both dictionary-encoded and - // plain-encoded data. - std::unordered_map> decoders_; - - void ConfigureDictionary(const DictionaryPage* page); - DecoderType* current_decoder_; + virtual int64_t Skip(int64_t num_rows_to_skip) = 0; }; -// ---------------------------------------------------------------------- -// Type column reader implementations - -template -inline int64_t TypedColumnReader::ReadValues(int64_t batch_size, T* out) { - int64_t num_decoded = current_decoder_->Decode(out, static_cast(batch_size)); - return num_decoded; -} - -template -inline int64_t TypedColumnReader::ReadValuesSpaced(int64_t batch_size, T* out, - int64_t null_count, - uint8_t* valid_bits, - int64_t valid_bits_offset) { - return current_decoder_->DecodeSpaced(out, static_cast(batch_size), - static_cast(null_count), valid_bits, - valid_bits_offset); -} - -template -inline int64_t TypedColumnReader::ReadBatch(int64_t batch_size, - int16_t* def_levels, - int16_t* rep_levels, T* values, - int64_t* values_read) { - // HasNext invokes ReadNewPage - if (!HasNext()) { - *values_read = 0; - return 0; - } - - // TODO(wesm): keep reading data pages until batch_size is reached, or the - // row group is finished - batch_size = std::min(batch_size, num_buffered_values_ - num_decoded_values_); - - int64_t num_def_levels = 0; - int64_t num_rep_levels = 0; +namespace internal { - int64_t values_to_read = 0; +static inline void DefinitionLevelsToBitmap( + const int16_t* def_levels, int64_t num_def_levels, const int16_t max_definition_level, + const int16_t max_repetition_level, int64_t* values_read, int64_t* null_count, + uint8_t* valid_bits, int64_t valid_bits_offset) { + // We assume here that valid_bits is large enough to accommodate the + // additional definition levels and the ones that have already been written + ::arrow::internal::BitmapWriter valid_bits_writer(valid_bits, valid_bits_offset, + valid_bits_offset + num_def_levels); - // If the field is required and non-repeated, there are no definition levels - if (descr_->max_definition_level() > 0 && def_levels) { - num_def_levels = ReadDefinitionLevels(batch_size, def_levels); - // TODO(wesm): this tallying of values-to-decode can be performed with better - // cache-efficiency if fused with the level decoding. - for (int64_t i = 0; i < num_def_levels; ++i) { - if (def_levels[i] == descr_->max_definition_level()) { - ++values_to_read; + // TODO(itaiin): As an interim solution we are splitting the code path here + // between repeated+flat column reads, and non-repeated+nested reads. + // Those paths need to be merged in the future + for (int i = 0; i < num_def_levels; ++i) { + if (def_levels[i] == max_definition_level) { + valid_bits_writer.Set(); + } else if (max_repetition_level > 0) { + // repetition+flat case + if (def_levels[i] == (max_definition_level - 1)) { + valid_bits_writer.Clear(); + *null_count += 1; + } else { + continue; + } + } else { + // non-repeated+nested case + if (def_levels[i] < max_definition_level) { + valid_bits_writer.Clear(); + *null_count += 1; + } else { + throw ParquetException("definition level exceeds maximum"); } } - } else { - // Required field, read all values - values_to_read = batch_size; - } - // Not present for non-repeated fields - if (descr_->max_repetition_level() > 0 && rep_levels) { - num_rep_levels = ReadRepetitionLevels(batch_size, rep_levels); - if (def_levels && num_def_levels != num_rep_levels) { - throw ParquetException("Number of decoded rep / def levels did not match"); - } + valid_bits_writer.Next(); } - - *values_read = ReadValues(values_to_read, values); - int64_t total_values = std::max(num_def_levels, *values_read); - ConsumeBufferedValues(total_values); - - return total_values; + valid_bits_writer.Finish(); + *values_read = valid_bits_writer.position(); } +} // namespace internal + namespace internal { // TODO(itaiin): another code path split to merge when the general case is done @@ -407,134 +251,13 @@ static inline bool HasSpacedValues(const ColumnDescriptor* descr) { } // namespace internal -template -inline int64_t TypedColumnReader::ReadBatchSpaced( - int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, T* values, - uint8_t* valid_bits, int64_t valid_bits_offset, int64_t* levels_read, - int64_t* values_read, int64_t* null_count_out) { - // HasNext invokes ReadNewPage - if (!HasNext()) { - *levels_read = 0; - *values_read = 0; - *null_count_out = 0; - return 0; - } - - int64_t total_values; - // TODO(wesm): keep reading data pages until batch_size is reached, or the - // row group is finished - batch_size = std::min(batch_size, num_buffered_values_ - num_decoded_values_); - - // If the field is required and non-repeated, there are no definition levels - if (descr_->max_definition_level() > 0) { - int64_t num_def_levels = ReadDefinitionLevels(batch_size, def_levels); - - // Not present for non-repeated fields - if (descr_->max_repetition_level() > 0) { - int64_t num_rep_levels = ReadRepetitionLevels(batch_size, rep_levels); - if (num_def_levels != num_rep_levels) { - throw ParquetException("Number of decoded rep / def levels did not match"); - } - } - - const bool has_spaced_values = internal::HasSpacedValues(descr_); - - int64_t null_count = 0; - if (!has_spaced_values) { - int values_to_read = 0; - for (int64_t i = 0; i < num_def_levels; ++i) { - if (def_levels[i] == descr_->max_definition_level()) { - ++values_to_read; - } - } - total_values = ReadValues(values_to_read, values); - for (int64_t i = 0; i < total_values; i++) { - ::arrow::BitUtil::SetBit(valid_bits, valid_bits_offset + i); - } - *values_read = total_values; - } else { - int16_t max_definition_level = descr_->max_definition_level(); - int16_t max_repetition_level = descr_->max_repetition_level(); - internal::DefinitionLevelsToBitmap(def_levels, num_def_levels, max_definition_level, - max_repetition_level, values_read, &null_count, - valid_bits, valid_bits_offset); - total_values = ReadValuesSpaced(*values_read, values, static_cast(null_count), - valid_bits, valid_bits_offset); - } - *levels_read = num_def_levels; - *null_count_out = null_count; - - } else { - // Required field, read all values - total_values = ReadValues(batch_size, values); - for (int64_t i = 0; i < total_values; i++) { - ::arrow::BitUtil::SetBit(valid_bits, valid_bits_offset + i); - } - *null_count_out = 0; - *levels_read = total_values; - } - - ConsumeBufferedValues(*levels_read); - return total_values; -} - -template -int64_t TypedColumnReader::Skip(int64_t num_rows_to_skip) { - int64_t rows_to_skip = num_rows_to_skip; - while (HasNext() && rows_to_skip > 0) { - // If the number of rows to skip is more than the number of undecoded values, skip the - // Page. - if (rows_to_skip > (num_buffered_values_ - num_decoded_values_)) { - rows_to_skip -= num_buffered_values_ - num_decoded_values_; - num_decoded_values_ = num_buffered_values_; - } else { - // We need to read this Page - // Jump to the right offset in the Page - int64_t batch_size = 1024; // ReadBatch with a smaller memory footprint - int64_t values_read = 0; - - std::shared_ptr vals = AllocateBuffer( - this->pool_, batch_size * type_traits::value_byte_size); - std::shared_ptr def_levels = - AllocateBuffer(this->pool_, batch_size * sizeof(int16_t)); - - std::shared_ptr rep_levels = - AllocateBuffer(this->pool_, batch_size * sizeof(int16_t)); - - do { - batch_size = std::min(batch_size, rows_to_skip); - values_read = ReadBatch(static_cast(batch_size), - reinterpret_cast(def_levels->mutable_data()), - reinterpret_cast(rep_levels->mutable_data()), - reinterpret_cast(vals->mutable_data()), &values_read); - rows_to_skip -= values_read; - } while (values_read > 0 && rows_to_skip > 0); - } - } - return num_rows_to_skip - rows_to_skip; -} - -// ---------------------------------------------------------------------- -// Template instantiations - -typedef TypedColumnReader BoolReader; -typedef TypedColumnReader Int32Reader; -typedef TypedColumnReader Int64Reader; -typedef TypedColumnReader Int96Reader; -typedef TypedColumnReader FloatReader; -typedef TypedColumnReader DoubleReader; -typedef TypedColumnReader ByteArrayReader; -typedef TypedColumnReader FixedLenByteArrayReader; - -PARQUET_EXTERN_TEMPLATE TypedColumnReader; -PARQUET_EXTERN_TEMPLATE TypedColumnReader; -PARQUET_EXTERN_TEMPLATE TypedColumnReader; -PARQUET_EXTERN_TEMPLATE TypedColumnReader; -PARQUET_EXTERN_TEMPLATE TypedColumnReader; -PARQUET_EXTERN_TEMPLATE TypedColumnReader; -PARQUET_EXTERN_TEMPLATE TypedColumnReader; -PARQUET_EXTERN_TEMPLATE TypedColumnReader; +using BoolReader = TypedColumnReader; +using Int32Reader = TypedColumnReader; +using Int64Reader = TypedColumnReader; +using Int96Reader = TypedColumnReader; +using FloatReader = TypedColumnReader; +using DoubleReader = TypedColumnReader; +using ByteArrayReader = TypedColumnReader; +using FixedLenByteArrayReader = TypedColumnReader; } // namespace parquet - -#endif // PARQUET_COLUMN_READER_H diff --git a/cpp/src/parquet/column_writer-test.cc b/cpp/src/parquet/column_writer-test.cc index 1f034b622719a..5db7a495e5a26 100644 --- a/cpp/src/parquet/column_writer-test.cc +++ b/cpp/src/parquet/column_writer-test.cc @@ -80,7 +80,8 @@ class TestPrimitiveWriter : public PrimitiveTypedTest { std::unique_ptr source(new InMemoryInputStream(buffer)); std::unique_ptr page_reader = PageReader::Open(std::move(source), num_rows, compression); - reader_.reset(new TypedColumnReader(this->descr_, std::move(page_reader))); + reader_ = std::static_pointer_cast>( + ColumnReader::Make(this->descr_, std::move(page_reader))); } std::shared_ptr> BuildWriter( @@ -260,7 +261,7 @@ class TestPrimitiveWriter : public PrimitiveTypedTest { int64_t values_read_; // Keep the reader alive as for ByteArray the lifetime of the ByteArray // content is bound to the reader. - std::unique_ptr> reader_; + std::shared_ptr> reader_; std::vector definition_levels_out_; std::vector repetition_levels_out_; diff --git a/cpp/src/parquet/encoding.cc b/cpp/src/parquet/encoding.cc index 3fd3ceca4c5e2..da630671f7903 100644 --- a/cpp/src/parquet/encoding.cc +++ b/cpp/src/parquet/encoding.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "arrow/builder.h" @@ -372,7 +373,7 @@ std::shared_ptr DictEncoderImpl::FlushValues() { int result_size = WriteIndices(buffer->mutable_data(), static_cast(EstimatedDataEncodedSize())); PARQUET_THROW_NOT_OK(buffer->Resize(result_size, false)); - return buffer; + return std::move(buffer); } template From ea63231110c07010df7090518dc678c0a37d1ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Wed, 30 Jan 2019 15:23:45 -0600 Subject: [PATCH 05/20] ARROW-4407: [C++] Cache compiler for CMake external projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensure that external projects use the same compiler that was first defined when invoking the `cmake` call. Author: François Saint-Jacques Closes #3515 from fsaintjacques/ARROW-4407-cmake-externaladd-compiler and squashes the following commits: ec6d5f20 fix zstd build 2 f3c0700a temp verbose appveyor 9085a7fc try windows fix 3f9296cd Clean thirdparty cmake f35abd49 ARROW-4407: Cache CMake compiler for external projects --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 197 ++++++++++---------- 1 file changed, 100 insertions(+), 97 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 13a6a472cc4e9..606b476992540 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -297,6 +297,31 @@ string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE) set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}") +if (NOT MSVC) + # Set -fPIC on all external projects + set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") + set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") +endif() + +# CC/CXX environment variables are captured on the first invocation of the +# builder (e.g make or ninja) instead of when CMake is invoked into to build +# directory. This leads to issues if the variables are exported in a subshell +# and the invocation of make/ninja is in distinct subshell without the same +# environment (CC/CXX). +set(EP_COMMON_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) + +# External projects are still able to override the following declarations. +# cmake command line will favor the last defined variable when a duplicate is +# encountered. This requires that `EP_COMMON_CMAKE_ARGS` is always the first +# argument. +set(EP_COMMON_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} + -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}) + if (NOT ARROW_VERBOSE_THIRDPARTY_BUILD) set(EP_LOG_OPTIONS LOG_CONFIGURE 1 @@ -309,12 +334,6 @@ else() set(Boost_DEBUG TRUE) endif() -if (NOT MSVC) - # Set -fPIC on all external projects - set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") - set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") -endif() - # Ensure that a default make is set if ("${MAKE}" STREQUAL "") if (NOT MSVC) @@ -498,11 +517,9 @@ if("${DOUBLE_CONVERSION_HOME}" STREQUAL "") set(DOUBLE_CONVERSION_INCLUDE_DIR "${DOUBLE_CONVERSION_PREFIX}/include") set(DOUBLE_CONVERSION_STATIC_LIB "${DOUBLE_CONVERSION_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}double-conversion${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(DOUBLE_CONVERSION_CMAKE_ARGS - "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" - "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}" - "-DCMAKE_INSTALL_PREFIX=${DOUBLE_CONVERSION_PREFIX}") + set(DOUBLE_CONVERSION_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${DOUBLE_CONVERSION_PREFIX}) + ExternalProject_Add(double-conversion_ep ${EP_LOG_OPTIONS} INSTALL_DIR ${DOUBLE_CONVERSION_PREFIX} @@ -540,8 +557,6 @@ if(ARROW_BUILD_TESTS OR (ARROW_USE_GLOG AND GLOG_HOME)) # gflags (formerly Googleflags) command line parsing if("${GFLAGS_HOME}" STREQUAL "") - set(GFLAGS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) - set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep") set(GFLAGS_HOME "${GFLAGS_PREFIX}") set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include") @@ -551,17 +566,14 @@ if(ARROW_BUILD_TESTS OR set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a") endif() set(GFLAGS_VENDORED 1) - set(GFLAGS_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + set(GFLAGS_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX} -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON -DBUILD_PACKAGING=OFF -DBUILD_TESTING=OFF -DBUILD_CONFIG_TESTS=OFF - -DINSTALL_HEADERS=ON - -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS} - -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS} - -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS}) + -DINSTALL_HEADERS=ON) ExternalProject_Add(gflags_ep URL ${GFLAGS_SOURCE_URL} @@ -595,13 +607,13 @@ endif() if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) if("${GTEST_HOME}" STREQUAL "") + set(GTEST_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) if(APPLE) - set(GTEST_CMAKE_CXX_FLAGS "-fPIC -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes") - elseif(NOT MSVC) - set(GTEST_CMAKE_CXX_FLAGS "-fPIC") + set(GTEST_CMAKE_CXX_FLAGS ${GTEST_CMAKE_CXX_FLAGS} + -DGTEST_USE_OWN_TR1_TUPLE=1 + -Wno-unused-value + -Wno-ignored-attributes) endif() - string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE) - set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}") set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") @@ -610,7 +622,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) set(GTEST_MAIN_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GTEST_VENDORED 1) - set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + set(GTEST_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX} -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) if (MSVC AND NOT ARROW_USE_STATIC_CRT) @@ -660,7 +672,7 @@ if(ARROW_BUILD_BENCHMARKS) endif() if(NOT MSVC) - set(GBENCHMARK_CMAKE_CXX_FLAGS "-fPIC -std=c++11 ${EP_CXX_FLAGS}") + set(GBENCHMARK_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++11") endif() if(APPLE) @@ -671,11 +683,10 @@ if(ARROW_BUILD_BENCHMARKS) set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include") set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GBENCHMARK_VENDORED 1) - set(GBENCHMARK_CMAKE_ARGS - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}" - "-DBENCHMARK_ENABLE_TESTING=OFF" - "-DCMAKE_CXX_FLAGS=${GBENCHMARK_CMAKE_CXX_FLAGS}") + set(GBENCHMARK_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${GBENCHMARK_PREFIX} + -DBENCHMARK_ENABLE_TESTING=OFF + -DCMAKE_CXX_FLAGS=${GBENCHMARK_CMAKE_CXX_FLAGS}) if (APPLE) set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON") endif() @@ -728,21 +739,22 @@ if (ARROW_IPC) ## Flatbuffers if("${FLATBUFFERS_HOME}" STREQUAL "") set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install") + set(FLATBUFFERS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) if (MSVC) - set(FLATBUFFERS_CMAKE_CXX_FLAGS /EHsc) - else() - set(FLATBUFFERS_CMAKE_CXX_FLAGS -fPIC) + set(FLATBUFFERS_CMAKE_CXX_FLAGS "/EHsc") endif() - # We always need to do release builds, otherwise flatc will not be installed. + + # RELEASE build is required for `flatc` to be installed. + set(FLATBUFFERS_BUILD_TYPE RELEASE) + set(FLATBUFFERS_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_BUILD_TYPE=${FLATBUFFERS_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${FLATBUFFERS_PREFIX} + -DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS} + -DFLATBUFFERS_BUILD_TESTS=OFF) + ExternalProject_Add(flatbuffers_ep URL ${FLATBUFFERS_SOURCE_URL} - CMAKE_ARGS - "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}" - "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}" - "-DFLATBUFFERS_BUILD_TESTS=OFF" - "-DCMAKE_BUILD_TYPE=RELEASE" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}" + CMAKE_ARGS ${FLATBUFFERS_CMAKE_ARGS} ${EP_LOG_OPTIONS}) set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include") @@ -866,11 +878,8 @@ if (ARROW_WITH_ZLIB) set(ZLIB_STATIC_LIB_NAME libz.a) endif() set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") - set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX} - -DCMAKE_C_FLAGS=${EP_C_FLAGS} - -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS} - -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS} -DBUILD_SHARED_LIBS=OFF) ADD_THIRDPARTY_LIB(zlib STATIC_LIB ${ZLIB_STATIC_LIB}) @@ -911,14 +920,10 @@ if (ARROW_WITH_SNAPPY) endif() if (WIN32) - set(SNAPPY_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}" - "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" - "-DCMAKE_AR=${CMAKE_AR}" - "-DCMAKE_RANLIB=${CMAKE_RANLIB}" - "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}") + set(SNAPPY_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_AR=${CMAKE_AR} + -DCMAKE_RANLIB=${CMAKE_RANLIB} + -DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}) set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt ./CMakeLists.txt && @@ -975,11 +980,7 @@ if (ARROW_WITH_BROTLI) set(BROTLI_STATIC_LIBRARY_ENC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}") set(BROTLI_STATIC_LIBRARY_DEC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}") set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BROTLI_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}" + set(BROTLI_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX} -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE} -DBUILD_SHARED_LIBS=OFF) @@ -1096,14 +1097,13 @@ if (ARROW_WITH_ZSTD) set(ZSTD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-install") set(ZSTD_INCLUDE_DIR "${ZSTD_PREFIX}/include") - set(ZSTD_CMAKE_ARGS - "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" - "-DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX}" - "-DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}" - "-DZSTD_BUILD_PROGRAMS=off" - "-DZSTD_BUILD_SHARED=off" - "-DZSTD_BUILD_STATIC=on" - "-DZSTD_MULTITHREAD_SUPPORT=off") + set(ZSTD_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX} + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} + -DZSTD_BUILD_PROGRAMS=off + -DZSTD_BUILD_SHARED=off + -DZSTD_BUILD_STATIC=on + -DZSTD_MULTITHREAD_SUPPORT=off) if (MSVC) set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/${CMAKE_INSTALL_LIBDIR}/zstd_static.lib") @@ -1115,8 +1115,10 @@ if (ARROW_WITH_ZSTD) # Only pass our C flags on Unix as on MSVC it leads to a # "incompatible command-line options" error set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} - "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${EP_C_FLAGS}") + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) endif() if(CMAKE_VERSION VERSION_LESS 3.7) @@ -1157,11 +1159,9 @@ if (ARROW_GANDIVA) set (RE2_INCLUDE_DIR "${RE2_PREFIX}/include") set (RE2_STATIC_LIB "${RE2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}re2${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(RE2_CMAKE_ARGS - "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" - "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}" - "-DCMAKE_INSTALL_PREFIX=${RE2_PREFIX}") + set(RE2_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${RE2_PREFIX}) + ExternalProject_Add(re2_ep ${EP_LOG_OPTIONS} INSTALL_DIR ${RE2_PREFIX} @@ -1200,9 +1200,17 @@ if (ARROW_ORC OR ARROW_FLIGHT OR ARROW_GANDIVA) set (PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include") set (PROTOBUF_STATIC_LIB "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") set (PROTOBUF_EXECUTABLE "${PROTOBUF_PREFIX}/bin/protoc") + set (PROTOBUF_CONFIGURE_ARGS "AR=${CMAKE_AR}" + "RANLIB=${CMAKE_RANLIB}" + "CC=${CMAKE_C_COMPILER}" + "CXX=${CMAKE_CXX_COMPILER}" + "--disable-shared" + "--prefix=${PROTOBUF_PREFIX}" + "CFLAGS=${EP_C_FLAGS}" + "CXXFLAGS=${EP_CXX_FLAGS}") ExternalProject_Add(protobuf_ep - CONFIGURE_COMMAND "./configure" "AR=${CMAKE_AR}" "RANLIB=${CMAKE_RANLIB}" "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" "--disable-shared" "--prefix=${PROTOBUF_PREFIX}" "CXXFLAGS=${EP_CXX_FLAGS}" + CONFIGURE_COMMAND "./configure" ${PROTOBUF_CONFIGURE_ARGS} BUILD_IN_SOURCE 1 URL ${PROTOBUF_SOURCE_URL} BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}" "${PROTOBUF_EXECUTABLE}" @@ -1244,9 +1252,7 @@ if (ARROW_FLIGHT) set(GRPC_STATIC_LIBRARY_GRPCPP "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GRPC_STATIC_LIBRARY_ADDRESS_SORTING "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}address_sorting${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GRPC_STATIC_LIBRARY_CARES "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/third_party/cares/cares/lib/${CMAKE_STATIC_LIBRARY_PREFIX}cares${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" + set(GRPC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} -DBUILD_SHARED_LIBS=OFF) @@ -1326,7 +1332,7 @@ if (ARROW_ORC) # Since LZ4 isn't installed, the header file is in ${LZ4_HOME}/lib instead of # ${LZ4_HOME}/include, which forces us to specify the include directory # manually as well. - set (ORC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + set (ORC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${ORC_PREFIX} -DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS} -DBUILD_LIBHDFSPP=OFF @@ -1388,24 +1394,21 @@ if (NOT THRIFT_FOUND) set(THRIFT_HOME "${THRIFT_PREFIX}") set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include") set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift") - set(THRIFT_CMAKE_ARGS "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" - "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" - "-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}" - "-DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib" - "-DBUILD_SHARED_LIBS=OFF" - "-DBUILD_TESTING=OFF" - "-DBUILD_EXAMPLES=OFF" - "-DBUILD_TUTORIALS=OFF" - "-DWITH_QT4=OFF" - "-DWITH_C_GLIB=OFF" - "-DWITH_JAVA=OFF" - "-DWITH_PYTHON=OFF" - "-DWITH_HASKELL=OFF" - "-DWITH_CPP=ON" - "-DWITH_STATIC_LIB=ON" - "-DWITH_LIBEVENT=OFF" - ) + set(THRIFT_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX} + -DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DBUILD_EXAMPLES=OFF + -DBUILD_TUTORIALS=OFF + -DWITH_QT4=OFF + -DWITH_C_GLIB=OFF + -DWITH_JAVA=OFF + -DWITH_PYTHON=OFF + -DWITH_HASKELL=OFF + -DWITH_CPP=ON + -DWITH_STATIC_LIB=ON + -DWITH_LIBEVENT=OFF) # Thrift also uses boost. Forward important boost settings if there were ones passed. if (DEFINED BOOST_ROOT) @@ -1538,7 +1541,7 @@ if (ARROW_USE_GLOG) set(GLOG_CMAKE_CXX_FLAGS "${GLOG_CMAKE_CXX_FLAGS} -mmacosx-version-min=10.9") endif() - set(GLOG_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + set(GLOG_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${GLOG_BUILD_DIR} -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF From 012f77a96880cff49f588ae1ff2f65d5105ee433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Wed, 30 Jan 2019 16:00:18 -0600 Subject: [PATCH 06/20] ARROW-4268: [C++] Native C type TypeTraits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add convenience support for native types, e.g. `TypeTraits` or `auto type = TypeTraits::singleton_type();`. This is very handy in tests. Author: François Saint-Jacques Closes #3514 from fsaintjacques/ARROW-4268-compile-c-type-traits and squashes the following commits: 946851c5 Make clang-format happy e55a7d72 Use CTypeTraits and refactor stl f9d7d0c7 Better param name dbb3114a ARROW-4268: Native C type TypeTraits --- cpp/src/arrow/stl.h | 44 +------- cpp/src/arrow/type-test.cc | 42 ++++---- cpp/src/arrow/type_traits.h | 197 +++++++++++++----------------------- cpp/src/arrow/util/macros.h | 1 + 4 files changed, 102 insertions(+), 182 deletions(-) diff --git a/cpp/src/arrow/stl.h b/cpp/src/arrow/stl.h index 5c632b31751c8..def496bccbc11 100644 --- a/cpp/src/arrow/stl.h +++ b/cpp/src/arrow/stl.h @@ -24,6 +24,7 @@ #include #include "arrow/type.h" +#include "arrow/type_traits.h" namespace arrow { @@ -31,40 +32,6 @@ class Schema; namespace stl { -/// Traits meta class to map standard C/C++ types to equivalent Arrow types. -template -struct ConversionTraits {}; - -#define ARROW_STL_CONVERSION(c_type, ArrowType_) \ - template <> \ - struct ConversionTraits { \ - static std::shared_ptr arrow_type() { \ - return std::make_shared(); \ - } \ - constexpr static bool nullable = false; \ - }; - -ARROW_STL_CONVERSION(bool, BooleanType) -ARROW_STL_CONVERSION(int8_t, Int8Type) -ARROW_STL_CONVERSION(int16_t, Int16Type) -ARROW_STL_CONVERSION(int32_t, Int32Type) -ARROW_STL_CONVERSION(int64_t, Int64Type) -ARROW_STL_CONVERSION(uint8_t, UInt8Type) -ARROW_STL_CONVERSION(uint16_t, UInt16Type) -ARROW_STL_CONVERSION(uint32_t, UInt32Type) -ARROW_STL_CONVERSION(uint64_t, UInt64Type) -ARROW_STL_CONVERSION(float, FloatType) -ARROW_STL_CONVERSION(double, DoubleType) -ARROW_STL_CONVERSION(std::string, StringType) - -template -struct ConversionTraits> { - static std::shared_ptr arrow_type() { - return list(ConversionTraits::arrow_type()); - } - constexpr static bool nullable = false; -}; - /// Build an arrow::Schema based upon the types defined in a std::tuple-like structure. /// /// While the type information is available at compile-time, we still need to add the @@ -82,8 +49,8 @@ struct SchemaFromTuple { const std::vector& names) { std::vector> ret = SchemaFromTuple::MakeSchemaRecursion(names); - std::shared_ptr type = ConversionTraits::arrow_type(); - ret.push_back(field(names[N - 1], type, ConversionTraits::nullable)); + std::shared_ptr type = CTypeTraits::type_singleton(); + ret.push_back(field(names[N - 1], type, false /* nullable */)); return ret; } @@ -111,9 +78,8 @@ struct SchemaFromTuple { const NamesTuple& names) { std::vector> ret = SchemaFromTuple::MakeSchemaRecursionT(names); - std::shared_ptr type = ConversionTraits::arrow_type(); - ret.push_back( - field(std::get(names), type, ConversionTraits::nullable)); + std::shared_ptr type = CTypeTraits::type_singleton(); + ret.push_back(field(std::get(names), type, false /* nullable */)); return ret; } diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc index 957c7632149f8..1bacbc937d5c6 100644 --- a/cpp/src/arrow/type-test.cc +++ b/cpp/src/arrow/type-test.cc @@ -27,6 +27,7 @@ #include "arrow/memory_pool.h" #include "arrow/test-util.h" #include "arrow/type.h" +#include "arrow/type_traits.h" #include "arrow/util/checked_cast.h" using std::shared_ptr; @@ -254,27 +255,34 @@ TEST_F(TestSchema, TestRemoveMetadata) { ASSERT_TRUE(new_schema->metadata() == nullptr); } -#define PRIMITIVE_TEST(KLASS, ENUM, NAME) \ - TEST(TypesTest, TestPrimitive_##ENUM) { \ - KLASS tp; \ - \ - ASSERT_EQ(tp.id(), Type::ENUM); \ - ASSERT_EQ(tp.ToString(), std::string(NAME)); \ +#define PRIMITIVE_TEST(KLASS, CTYPE, ENUM, NAME) \ + TEST(TypesTest, ARROW_CONCAT(TestPrimitive_, ENUM)) { \ + KLASS tp; \ + \ + ASSERT_EQ(tp.id(), Type::ENUM); \ + ASSERT_EQ(tp.ToString(), std::string(NAME)); \ + \ + using CType = TypeTraits::CType; \ + static_assert(std::is_same::value, "Not the same c-type!"); \ + \ + using DerivedArrowType = CTypeTraits::ArrowType; \ + static_assert(std::is_same::value, \ + "Not the same arrow-type!"); \ } -PRIMITIVE_TEST(Int8Type, INT8, "int8") -PRIMITIVE_TEST(Int16Type, INT16, "int16") -PRIMITIVE_TEST(Int32Type, INT32, "int32") -PRIMITIVE_TEST(Int64Type, INT64, "int64") -PRIMITIVE_TEST(UInt8Type, UINT8, "uint8") -PRIMITIVE_TEST(UInt16Type, UINT16, "uint16") -PRIMITIVE_TEST(UInt32Type, UINT32, "uint32") -PRIMITIVE_TEST(UInt64Type, UINT64, "uint64") +PRIMITIVE_TEST(Int8Type, int8_t, INT8, "int8"); +PRIMITIVE_TEST(Int16Type, int16_t, INT16, "int16"); +PRIMITIVE_TEST(Int32Type, int32_t, INT32, "int32"); +PRIMITIVE_TEST(Int64Type, int64_t, INT64, "int64"); +PRIMITIVE_TEST(UInt8Type, uint8_t, UINT8, "uint8"); +PRIMITIVE_TEST(UInt16Type, uint16_t, UINT16, "uint16"); +PRIMITIVE_TEST(UInt32Type, uint32_t, UINT32, "uint32"); +PRIMITIVE_TEST(UInt64Type, uint64_t, UINT64, "uint64"); -PRIMITIVE_TEST(FloatType, FLOAT, "float") -PRIMITIVE_TEST(DoubleType, DOUBLE, "double") +PRIMITIVE_TEST(FloatType, float, FLOAT, "float"); +PRIMITIVE_TEST(DoubleType, double, DOUBLE, "double"); -PRIMITIVE_TEST(BooleanType, BOOL, "bool") +PRIMITIVE_TEST(BooleanType, bool, BOOL, "bool"); TEST(TestBinaryType, ToString) { BinaryType t1; diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h index edb8ca166f6ee..fd1d52a370f24 100644 --- a/cpp/src/arrow/type_traits.h +++ b/cpp/src/arrow/type_traits.h @@ -19,7 +19,9 @@ #define ARROW_TYPE_TRAITS_H #include +#include #include +#include #include "arrow/type_fwd.h" #include "arrow/util/bit-util.h" @@ -33,6 +35,9 @@ namespace arrow { template struct TypeTraits {}; +template +struct CTypeTraits {}; + template <> struct TypeTraits { using ArrayType = NullArray; @@ -41,102 +46,61 @@ struct TypeTraits { }; template <> -struct TypeTraits { - using ArrayType = UInt8Array; - using BuilderType = UInt8Builder; - using TensorType = UInt8Tensor; - static constexpr int64_t bytes_required(int64_t elements) { return elements; } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return uint8(); } -}; - -template <> -struct TypeTraits { - using ArrayType = Int8Array; - using BuilderType = Int8Builder; - using TensorType = Int8Tensor; - static constexpr int64_t bytes_required(int64_t elements) { return elements; } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return int8(); } -}; - -template <> -struct TypeTraits { - using ArrayType = UInt16Array; - using BuilderType = UInt16Builder; - using TensorType = UInt16Tensor; - - static constexpr int64_t bytes_required(int64_t elements) { - return elements * sizeof(uint16_t); - } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return uint16(); } -}; - -template <> -struct TypeTraits { - using ArrayType = Int16Array; - using BuilderType = Int16Builder; - using TensorType = Int16Tensor; - - static constexpr int64_t bytes_required(int64_t elements) { - return elements * sizeof(int16_t); - } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return int16(); } -}; - -template <> -struct TypeTraits { - using ArrayType = UInt32Array; - using BuilderType = UInt32Builder; - using TensorType = UInt32Tensor; - - static constexpr int64_t bytes_required(int64_t elements) { - return elements * sizeof(uint32_t); - } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return uint32(); } -}; - -template <> -struct TypeTraits { - using ArrayType = Int32Array; - using BuilderType = Int32Builder; - using TensorType = Int32Tensor; - - static constexpr int64_t bytes_required(int64_t elements) { - return elements * sizeof(int32_t); - } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return int32(); } -}; - -template <> -struct TypeTraits { - using ArrayType = UInt64Array; - using BuilderType = UInt64Builder; - using TensorType = UInt64Tensor; +struct TypeTraits { + using ArrayType = BooleanArray; + using BuilderType = BooleanBuilder; + using CType = bool; static constexpr int64_t bytes_required(int64_t elements) { - return elements * sizeof(uint64_t); + return BitUtil::BytesForBits(elements); } constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return uint64(); } + static inline std::shared_ptr type_singleton() { return boolean(); } }; template <> -struct TypeTraits { - using ArrayType = Int64Array; - using BuilderType = Int64Builder; - using TensorType = Int64Tensor; +struct CTypeTraits : public TypeTraits { + using ArrowType = BooleanType; +}; + +#define PRIMITIVE_TYPE_TRAITS_DEF_(CType_, ArrowType_, ArrowArrayType, ArrowBuilderType, \ + ArrowTensorType, SingletonFn) \ + template <> \ + struct TypeTraits { \ + using ArrayType = ArrowArrayType; \ + using BuilderType = ArrowBuilderType; \ + using TensorType = ArrowTensorType; \ + using CType = CType_; \ + static constexpr int64_t bytes_required(int64_t elements) { \ + return elements * sizeof(CType_); \ + } \ + constexpr static bool is_parameter_free = true; \ + static inline std::shared_ptr type_singleton() { return SingletonFn(); } \ + }; \ + \ + template <> \ + struct CTypeTraits : public TypeTraits { \ + using ArrowType = ArrowType_; \ + }; - static constexpr int64_t bytes_required(int64_t elements) { - return elements * sizeof(int64_t); - } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return int64(); } -}; +#define PRIMITIVE_TYPE_TRAITS_DEF(CType, ArrowShort, SingletonFn) \ + PRIMITIVE_TYPE_TRAITS_DEF_( \ + CType, ARROW_CONCAT(ArrowShort, Type), ARROW_CONCAT(ArrowShort, Array), \ + ARROW_CONCAT(ArrowShort, Builder), ARROW_CONCAT(ArrowShort, Tensor), SingletonFn) + +PRIMITIVE_TYPE_TRAITS_DEF(uint8_t, UInt8, uint8) +PRIMITIVE_TYPE_TRAITS_DEF(int8_t, Int8, int8) +PRIMITIVE_TYPE_TRAITS_DEF(uint16_t, UInt16, uint16) +PRIMITIVE_TYPE_TRAITS_DEF(int16_t, Int16, int16) +PRIMITIVE_TYPE_TRAITS_DEF(uint32_t, UInt32, uint32) +PRIMITIVE_TYPE_TRAITS_DEF(int32_t, Int32, int32) +PRIMITIVE_TYPE_TRAITS_DEF(uint64_t, UInt64, uint64) +PRIMITIVE_TYPE_TRAITS_DEF(int64_t, Int64, int64) +PRIMITIVE_TYPE_TRAITS_DEF(float, Float, float32) +PRIMITIVE_TYPE_TRAITS_DEF(double, Double, float64) + +#undef PRIMITIVE_TYPE_TRAITS_DEF +#undef PRIMITIVE_TYPE_TRAITS_DEF_ template <> struct TypeTraits { @@ -208,32 +172,6 @@ struct TypeTraits { static inline std::shared_ptr type_singleton() { return float16(); } }; -template <> -struct TypeTraits { - using ArrayType = FloatArray; - using BuilderType = FloatBuilder; - using TensorType = FloatTensor; - - static constexpr int64_t bytes_required(int64_t elements) { - return static_cast(elements * sizeof(float)); - } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return float32(); } -}; - -template <> -struct TypeTraits { - using ArrayType = DoubleArray; - using BuilderType = DoubleBuilder; - using TensorType = DoubleTensor; - - static constexpr int64_t bytes_required(int64_t elements) { - return static_cast(elements * sizeof(double)); - } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return float64(); } -}; - template <> struct TypeTraits { using ArrayType = Decimal128Array; @@ -241,18 +179,6 @@ struct TypeTraits { constexpr static bool is_parameter_free = false; }; -template <> -struct TypeTraits { - using ArrayType = BooleanArray; - using BuilderType = BooleanBuilder; - - static constexpr int64_t bytes_required(int64_t elements) { - return BitUtil::BytesForBits(elements); - } - constexpr static bool is_parameter_free = true; - static inline std::shared_ptr type_singleton() { return boolean(); } -}; - template <> struct TypeTraits { using ArrayType = StringArray; @@ -261,6 +187,16 @@ struct TypeTraits { static inline std::shared_ptr type_singleton() { return utf8(); } }; +template <> +struct CTypeTraits : public TypeTraits { + using ArrowType = StringType; +}; + +template <> +struct CTypeTraits : public TypeTraits { + using ArrowType = StringType; +}; + template <> struct TypeTraits { using ArrayType = BinaryArray; @@ -283,6 +219,15 @@ struct TypeTraits { constexpr static bool is_parameter_free = false; }; +template +struct CTypeTraits> : public TypeTraits { + using ArrowType = ListType; + + static inline std::shared_ptr type_singleton() { + return list(CTypeTraits::type_singleton()); + } +}; + template <> struct TypeTraits { using ArrayType = StructArray; diff --git a/cpp/src/arrow/util/macros.h b/cpp/src/arrow/util/macros.h index ab258252695ab..5f1934d732ca7 100644 --- a/cpp/src/arrow/util/macros.h +++ b/cpp/src/arrow/util/macros.h @@ -19,6 +19,7 @@ #define ARROW_UTIL_MACROS_H #define ARROW_STRINGIFY(x) #x +#define ARROW_CONCAT(x, y) x##y // From Google gutil #ifndef ARROW_DISALLOW_COPY_AND_ASSIGN From 73792817a039a16c179cbb2716ab2cf0c918f77e Mon Sep 17 00:00:00 2001 From: Anurag Khandelwal Date: Wed, 30 Jan 2019 17:44:57 -0800 Subject: [PATCH 07/20] ARROW-4422: [Plasma] Enforce memory limit in plasma, rather than relying on dlmalloc_set_footprint_limit https://issues.apache.org/jira/browse/ARROW-4422 This implementation imposes memory limit at Plasma by tracking the number of bytes allocated and freed using malloc and free calls. Whenever the allocation reaches the set limit, we fail any subsequent allocations (by returning NULL from malloc). This allows Plasma to not be tied to dlmalloc, and also provides more accurate tracking of memory allocation/capacity. cc @pcmoritz Author: Anurag Khandelwal Closes #3526 from anuragkh/plasma_track_memory_use and squashes the following commits: 182e8a401 Replace dlmalloc calls with PlasmaAllocator, which internally calls dlmalloc --- cpp/src/plasma/CMakeLists.txt | 1 + cpp/src/plasma/eviction_policy.cc | 23 ++++------- cpp/src/plasma/eviction_policy.h | 2 - cpp/src/plasma/plasma.cc | 7 +--- cpp/src/plasma/plasma.h | 3 -- cpp/src/plasma/plasma_allocator.cc | 56 ++++++++++++++++++++++++++ cpp/src/plasma/plasma_allocator.h | 64 ++++++++++++++++++++++++++++++ cpp/src/plasma/store.cc | 45 ++++++++------------- cpp/src/plasma/store.h | 3 +- 9 files changed, 148 insertions(+), 56 deletions(-) create mode 100644 cpp/src/plasma/plasma_allocator.cc create mode 100644 cpp/src/plasma/plasma_allocator.h diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt index 566066463fffe..53af8c531aad8 100644 --- a/cpp/src/plasma/CMakeLists.txt +++ b/cpp/src/plasma/CMakeLists.txt @@ -76,6 +76,7 @@ set(PLASMA_SRCS io.cc malloc.cc plasma.cc + plasma_allocator.cc protocol.cc thirdparty/ae/ae.c) diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc index 4fb0cce81ecee..da5df5a36ddd4 100644 --- a/cpp/src/plasma/eviction_policy.cc +++ b/cpp/src/plasma/eviction_policy.cc @@ -16,6 +16,7 @@ // under the License. #include "plasma/eviction_policy.h" +#include "plasma/plasma_allocator.h" #include @@ -48,8 +49,7 @@ int64_t LRUCache::ChooseObjectsToEvict(int64_t num_bytes_required, return bytes_evicted; } -EvictionPolicy::EvictionPolicy(PlasmaStoreInfo* store_info) - : memory_used_(0), store_info_(store_info) {} +EvictionPolicy::EvictionPolicy(PlasmaStoreInfo* store_info) : store_info_(store_info) {} int64_t EvictionPolicy::ChooseObjectsToEvict(int64_t num_bytes_required, std::vector* objects_to_evict) { @@ -59,33 +59,29 @@ int64_t EvictionPolicy::ChooseObjectsToEvict(int64_t num_bytes_required, for (auto& object_id : *objects_to_evict) { cache_.Remove(object_id); } - // Update the number of bytes used. - memory_used_ -= bytes_evicted; - ARROW_CHECK(memory_used_ >= 0); return bytes_evicted; } void EvictionPolicy::ObjectCreated(const ObjectID& object_id) { auto entry = store_info_->objects[object_id].get(); cache_.Add(object_id, entry->data_size + entry->metadata_size); - int64_t size = entry->data_size + entry->metadata_size; - memory_used_ += size; - ARROW_CHECK(memory_used_ <= store_info_->memory_capacity); } bool EvictionPolicy::RequireSpace(int64_t size, std::vector* objects_to_evict) { // Check if there is enough space to create the object. - int64_t required_space = memory_used_ + size - store_info_->memory_capacity; + int64_t required_space = + PlasmaAllocator::Allocated() + size - PlasmaAllocator::GetFootprintLimit(); // Try to free up at least as much space as we need right now but ideally // up to 20% of the total capacity. - int64_t space_to_free = std::max(required_space, store_info_->memory_capacity / 5); + int64_t space_to_free = + std::max(required_space, PlasmaAllocator::GetFootprintLimit() / 5); ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects"; // Choose some objects to evict, and update the return pointers. int64_t num_bytes_evicted = ChooseObjectsToEvict(space_to_free, objects_to_evict); ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting " << objects_to_evict->size() << " objects to free up " << num_bytes_evicted << " bytes. The number of bytes in use (before " - << "this eviction) is " << (memory_used_ + num_bytes_evicted) << "."; + << "this eviction) is " << PlasmaAllocator::Allocated() << "."; return num_bytes_evicted >= required_space && num_bytes_evicted > 0; } @@ -105,11 +101,6 @@ void EvictionPolicy::EndObjectAccess(const ObjectID& object_id, void EvictionPolicy::RemoveObject(const ObjectID& object_id) { // If the object is in the LRU cache, remove it. cache_.Remove(object_id); - - auto entry = store_info_->objects[object_id].get(); - int64_t size = entry->data_size + entry->metadata_size; - ARROW_CHECK(memory_used_ >= size); - memory_used_ -= size; } } // namespace plasma diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h index bbd3fc4320356..68342ae102f3e 100644 --- a/cpp/src/plasma/eviction_policy.h +++ b/cpp/src/plasma/eviction_policy.h @@ -126,8 +126,6 @@ class EvictionPolicy { void RemoveObject(const ObjectID& object_id); private: - /// The amount of memory (in bytes) currently being used. - int64_t memory_used_; /// Pointer to the plasma store info. PlasmaStoreInfo* store_info_; /// Datastructure for the LRU cache. diff --git a/cpp/src/plasma/plasma.cc b/cpp/src/plasma/plasma.cc index a1749d0865d50..e1c10369dc6ef 100644 --- a/cpp/src/plasma/plasma.cc +++ b/cpp/src/plasma/plasma.cc @@ -23,20 +23,17 @@ #include "plasma/common.h" #include "plasma/common_generated.h" +#include "plasma/plasma_allocator.h" #include "plasma/protocol.h" namespace fb = plasma::flatbuf; namespace plasma { -extern "C" { -void dlfree(void* mem); -} - ObjectTableEntry::ObjectTableEntry() : pointer(nullptr), ref_count(0) {} ObjectTableEntry::~ObjectTableEntry() { - dlfree(pointer); + PlasmaAllocator::Free(pointer, data_size + metadata_size); pointer = nullptr; } diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h index 3d3eca19acc63..e23969d05ff3b 100644 --- a/cpp/src/plasma/plasma.h +++ b/cpp/src/plasma/plasma.h @@ -104,9 +104,6 @@ enum class ObjectStatus : int { struct PlasmaStoreInfo { /// Objects that are in the Plasma store. ObjectTable objects; - /// The amount of memory (in bytes) that we allow to be allocated in the - /// store. - int64_t memory_capacity; /// Boolean flag indicating whether to start the object store with hugepages /// support enabled. Huge pages are substantially larger than normal memory /// pages (e.g. 2MB or 1GB instead of 4KB) and using them can reduce diff --git a/cpp/src/plasma/plasma_allocator.cc b/cpp/src/plasma/plasma_allocator.cc new file mode 100644 index 0000000000000..b67eeea404bce --- /dev/null +++ b/cpp/src/plasma/plasma_allocator.cc @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "plasma/malloc.h" +#include "plasma/plasma_allocator.h" + +namespace plasma { + +extern "C" { +void* dlmemalign(size_t alignment, size_t bytes); +void dlfree(void* mem); +} + +int64_t PlasmaAllocator::footprint_limit_ = 0; +int64_t PlasmaAllocator::allocated_ = 0; + +void* PlasmaAllocator::Memalign(size_t alignment, size_t bytes) { + if (allocated_ + static_cast(bytes) > footprint_limit_) { + return nullptr; + } + void* mem = dlmemalign(alignment, bytes); + ARROW_CHECK(mem); + allocated_ += bytes; + return mem; +} + +void PlasmaAllocator::Free(void* mem, size_t bytes) { + dlfree(mem); + allocated_ -= bytes; +} + +void PlasmaAllocator::SetFootprintLimit(size_t bytes) { + footprint_limit_ = static_cast(bytes); +} + +int64_t PlasmaAllocator::GetFootprintLimit() { return footprint_limit_; } + +int64_t PlasmaAllocator::Allocated() { return allocated_; } + +} // namespace plasma diff --git a/cpp/src/plasma/plasma_allocator.h b/cpp/src/plasma/plasma_allocator.h new file mode 100644 index 0000000000000..d9d4cc0ecbe0c --- /dev/null +++ b/cpp/src/plasma/plasma_allocator.h @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef PLASMA_ALLOCATOR_H +#define PLASMA_ALLOCATOR_H + +#include +#include + +namespace plasma { + +class PlasmaAllocator { + public: + /// Allocates size bytes and returns a pointer to the allocated memory. The + /// memory address will be a multiple of alignment, which must be a power of two. + /// + /// \param alignment Memory alignment. + /// \param bytes Number of bytes. + /// \return Pointer to allocated memory. + static void* Memalign(size_t alignment, size_t bytes); + + /// Frees the memory space pointed to by mem, which must have been returned by + /// a previous call to Memalign() + /// + /// \param mem Pointer to memory to free. + /// \param bytes Number of bytes to be freed. + static void Free(void* mem, size_t bytes); + + /// Sets the memory footprint limit for Plasma. + /// + /// \param bytes Plasma memory footprint limit in bytes. + static void SetFootprintLimit(size_t bytes); + + /// Get the memory footprint limit for Plasma. + /// + /// \return Plasma memory footprint limit in bytes. + static int64_t GetFootprintLimit(); + + /// Get the number of bytes allocated by Plasma so far. + /// \return Number of bytes allocated by Plasma so far. + static int64_t Allocated(); + + private: + static int64_t allocated_; + static int64_t footprint_limit_; +}; + +} // namespace plasma + +#endif // ARROW_PLASMA_ALLOCATOR_H diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc index 343ccf5b886f2..f84e89077ca69 100644 --- a/cpp/src/plasma/store.cc +++ b/cpp/src/plasma/store.cc @@ -59,6 +59,7 @@ #include "plasma/fling.h" #include "plasma/io.h" #include "plasma/malloc.h" +#include "plasma/plasma_allocator.h" #include "plasma/protocol.h" #ifdef PLASMA_CUDA @@ -76,13 +77,6 @@ namespace fb = plasma::flatbuf; namespace plasma { -extern "C" { -void* dlmalloc(size_t bytes); -void* dlmemalign(size_t alignment, size_t bytes); -void dlfree(void* mem); -size_t dlmalloc_set_footprint_limit(size_t bytes); -} - struct GetRequest { GetRequest(Client* client, const std::vector& object_ids); /// The client that called get. @@ -114,10 +108,8 @@ GetRequest::GetRequest(Client* client, const std::vector& object_ids) Client::Client(int fd) : fd(fd), notification_fd(-1) {} -PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory, std::string directory, - bool hugepages_enabled) +PlasmaStore::PlasmaStore(EventLoop* loop, std::string directory, bool hugepages_enabled) : loop_(loop), eviction_policy_(&store_info_) { - store_info_.memory_capacity = system_memory; store_info_.directory = directory; store_info_.hugepages_enabled = hugepages_enabled; #ifdef PLASMA_CUDA @@ -173,7 +165,7 @@ PlasmaError PlasmaStore::CreateObject(const ObjectID& object_id, int64_t data_si } #endif while (true) { - // Allocate space for the new object. We use dlmemalign instead of dlmalloc + // Allocate space for the new object. We use memalign instead of malloc // in order to align the allocated region to a 64-byte boundary. This is not // strictly necessary, but it is an optimization that could speed up the // computation of a hash of the data (see compute_object_hash_parallel in @@ -181,8 +173,8 @@ PlasmaError PlasmaStore::CreateObject(const ObjectID& object_id, int64_t data_si // it is not guaranteed that the corresponding pointer in the client will be // 64-byte aligned, but in practice it often will be. if (device_num == 0) { - pointer = - reinterpret_cast(dlmemalign(kBlockSize, data_size + metadata_size)); + pointer = reinterpret_cast( + PlasmaAllocator::Memalign(kBlockSize, data_size + metadata_size)); if (pointer == nullptr) { // Tell the eviction policy how much space we need to create this object. std::vector objects_to_evict; @@ -886,7 +878,7 @@ Status PlasmaStore::ProcessMessage(Client* client) { SubscribeToUpdates(client); break; case fb::MessageType::PlasmaConnectRequest: { - HANDLE_SIGPIPE(SendConnectReply(client->fd, store_info_.memory_capacity), + HANDLE_SIGPIPE(SendConnectReply(client->fd, PlasmaAllocator::GetFootprintLimit()), client->fd); } break; case fb::MessageType::PlasmaDisconnectClient: @@ -904,23 +896,23 @@ class PlasmaStoreRunner { public: PlasmaStoreRunner() {} - void Start(char* socket_name, int64_t system_memory, std::string directory, - bool hugepages_enabled) { + void Start(char* socket_name, std::string directory, bool hugepages_enabled) { // Create the event loop. loop_.reset(new EventLoop); - store_.reset( - new PlasmaStore(loop_.get(), system_memory, directory, hugepages_enabled)); + store_.reset(new PlasmaStore(loop_.get(), directory, hugepages_enabled)); plasma_config = store_->GetPlasmaStoreInfo(); // We are using a single memory-mapped file by mallocing and freeing a single // large amount of space up front. According to the documentation, // dlmalloc might need up to 128*sizeof(size_t) bytes for internal // bookkeeping. - void* pointer = plasma::dlmemalign(kBlockSize, system_memory - 256 * sizeof(size_t)); + void* pointer = plasma::PlasmaAllocator::Memalign( + kBlockSize, PlasmaAllocator::GetFootprintLimit() - 256 * sizeof(size_t)); ARROW_CHECK(pointer != nullptr); // This will unmap the file, but the next one created will be as large // as this one (this is an implementation detail of dlmalloc). - plasma::dlfree(pointer); + plasma::PlasmaAllocator::Free( + pointer, PlasmaAllocator::GetFootprintLimit() - 256 * sizeof(size_t)); int socket = BindIpcSock(socket_name, true); // TODO(pcm): Check return value. @@ -955,7 +947,7 @@ void HandleSignal(int signal) { } } -void StartServer(char* socket_name, int64_t system_memory, std::string plasma_directory, +void StartServer(char* socket_name, std::string plasma_directory, bool hugepages_enabled) { // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write // to a client that has already died, the store could die. @@ -963,7 +955,7 @@ void StartServer(char* socket_name, int64_t system_memory, std::string plasma_di g_runner.reset(new PlasmaStoreRunner()); signal(SIGTERM, HandleSignal); - g_runner->Start(socket_name, system_memory, plasma_directory, hugepages_enabled); + g_runner->Start(socket_name, plasma_directory, hugepages_enabled); } } // namespace plasma @@ -992,11 +984,8 @@ int main(int argc, char* argv[]) { char extra; int scanned = sscanf(optarg, "%" SCNd64 "%c", &system_memory, &extra); ARROW_CHECK(scanned == 1); - // Set system memory, potentially rounding it to a page size - // Also make it so dlmalloc fails if we try to request more memory than - // is available. - system_memory = - plasma::dlmalloc_set_footprint_limit(static_cast(system_memory)); + // Set system memory capacity + plasma::PlasmaAllocator::SetFootprintLimit(static_cast(system_memory)); ARROW_LOG(INFO) << "Allowing the Plasma store to use up to " << static_cast(system_memory) / 1000000000 << "GB of memory."; @@ -1052,7 +1041,7 @@ int main(int argc, char* argv[]) { } #endif ARROW_LOG(DEBUG) << "starting server listening on " << socket_name; - plasma::StartServer(socket_name, system_memory, plasma_directory, hugepages_enabled); + plasma::StartServer(socket_name, plasma_directory, hugepages_enabled); plasma::g_runner->Shutdown(); plasma::g_runner = nullptr; diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h index 9866e74576f00..a5c586b7f53f0 100644 --- a/cpp/src/plasma/store.h +++ b/cpp/src/plasma/store.h @@ -75,8 +75,7 @@ class PlasmaStore { using NotificationMap = std::unordered_map; // TODO: PascalCase PlasmaStore methods. - PlasmaStore(EventLoop* loop, int64_t system_memory, std::string directory, - bool hugetlbfs_enabled); + PlasmaStore(EventLoop* loop, std::string directory, bool hugetlbfs_enabled); ~PlasmaStore(); From aa43784fde1ea632a26e6cbd590a0b6f1f6bfeca Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 31 Jan 2019 13:12:09 +0100 Subject: [PATCH 08/20] ARROW-4285: [Python] Use proper builder interface for serialization The UnionBuilder API is still pretty rough, feedback on how it should look is appreciated. It also raises the question on what the best way to do type inference (i.e. which type will the UnionArray have?). Having this in the builder would be super useful though also for other applications where the schema is not known in advance. Author: Philipp Moritz Closes #3423 from pcmoritz/union-builder and squashes the following commits: 0c6cedab add comment 9d5fcc21 lint 39982cd8 remove infer_type 745ed588 more comments d5e23003 comments 9697e692 dynamic_cast -> checked_cast 0ae5b66c fixes a82720cc add documentation 91d08e3d remove macro ff7e052a use VisitIterable 2b7e4b9a more cleanups 1db3c218 cleanups 7b41b058 starting to refactor UnionBuilder 74888345 use enum for types, type constructors 80c98383 clean up history for union builder branch --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/array/builder_nested.cc | 24 +- cpp/src/arrow/array/builder_union.cc | 60 +++ cpp/src/arrow/array/builder_union.h | 89 ++++ cpp/src/arrow/python/deserialize.cc | 201 ++++----- cpp/src/arrow/python/serialize.cc | 575 +++++++++----------------- cpp/src/arrow/python/serialize.h | 22 + 7 files changed, 488 insertions(+), 484 deletions(-) create mode 100644 cpp/src/arrow/array/builder_union.cc create mode 100644 cpp/src/arrow/array/builder_union.h diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 22ce6e913368e..838e3966ee895 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -73,6 +73,7 @@ set(ARROW_SRCS array/builder_dict.cc array/builder_nested.cc array/builder_primitive.cc + array/builder_union.cc buffer.cc compare.cc diff --git a/cpp/src/arrow/array/builder_nested.cc b/cpp/src/arrow/array/builder_nested.cc index 2f600cd9b9228..46637713c3e0f 100644 --- a/cpp/src/arrow/array/builder_nested.cc +++ b/cpp/src/arrow/array/builder_nested.cc @@ -99,6 +99,12 @@ Status ListBuilder::FinishInternal(std::shared_ptr* out) { RETURN_NOT_OK(value_builder_->FinishInternal(&items)); } + // If the type has not been specified in the constructor, infer it + // This is the case if the value_builder contains a DenseUnionBuilder + if (!arrow::internal::checked_cast(*type_).value_type()) { + type_ = std::static_pointer_cast( + std::make_shared(value_builder_->type())); + } std::shared_ptr null_bitmap; RETURN_NOT_OK(null_bitmap_builder_.Finish(&null_bitmap)); *out = ArrayData::Make(type_, length_, {null_bitmap, offsets}, null_count_); @@ -138,17 +144,29 @@ void StructBuilder::Reset() { Status StructBuilder::FinishInternal(std::shared_ptr* out) { std::shared_ptr null_bitmap; RETURN_NOT_OK(null_bitmap_builder_.Finish(&null_bitmap)); - *out = ArrayData::Make(type_, length_, {null_bitmap}, null_count_); - (*out)->child_data.resize(children_.size()); + std::vector> child_data(children_.size()); for (size_t i = 0; i < children_.size(); ++i) { if (length_ == 0) { // Try to make sure the child buffers are initialized RETURN_NOT_OK(children_[i]->Resize(0)); } - RETURN_NOT_OK(children_[i]->FinishInternal(&(*out)->child_data[i])); + RETURN_NOT_OK(children_[i]->FinishInternal(&child_data[i])); } + // If the type has not been specified in the constructor, infer it + // This is the case if one of the children contains a DenseUnionBuilder + if (!type_) { + std::vector> fields; + for (const auto& field_builder : children_) { + fields.push_back(field("", field_builder->type())); + } + type_ = struct_(fields); + } + + *out = ArrayData::Make(type_, length_, {null_bitmap}, null_count_); + (*out)->child_data = std::move(child_data); + capacity_ = length_ = null_count_ = 0; return Status::OK(); } diff --git a/cpp/src/arrow/array/builder_union.cc b/cpp/src/arrow/array/builder_union.cc new file mode 100644 index 0000000000000..f51b7d7f0203a --- /dev/null +++ b/cpp/src/arrow/array/builder_union.cc @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/array/builder_union.h" + +#include + +#include "arrow/util/logging.h" + +namespace arrow { + +DenseUnionBuilder::DenseUnionBuilder(MemoryPool* pool, + const std::shared_ptr& type) + : ArrayBuilder(type, pool), types_builder_(pool), offsets_builder_(pool) {} + +Status DenseUnionBuilder::FinishInternal(std::shared_ptr* out) { + std::shared_ptr types; + RETURN_NOT_OK(types_builder_.Finish(&types)); + std::shared_ptr offsets; + RETURN_NOT_OK(offsets_builder_.Finish(&offsets)); + + std::shared_ptr null_bitmap; + RETURN_NOT_OK(null_bitmap_builder_.Finish(&null_bitmap)); + + std::vector> fields; + std::vector> child_data(children_.size()); + std::vector type_ids; + for (size_t i = 0; i < children_.size(); ++i) { + std::shared_ptr data; + RETURN_NOT_OK(children_[i]->FinishInternal(&data)); + child_data[i] = data; + fields.push_back(field(field_names_[i], children_[i]->type())); + type_ids.push_back(static_cast(i)); + } + + // If the type has not been specified in the constructor, infer it + if (!type_) { + type_ = union_(fields, type_ids, UnionMode::DENSE); + } + + *out = ArrayData::Make(type_, length(), {null_bitmap, types, offsets}, null_count_); + (*out)->child_data = std::move(child_data); + return Status::OK(); +} + +} // namespace arrow diff --git a/cpp/src/arrow/array/builder_union.h b/cpp/src/arrow/array/builder_union.h new file mode 100644 index 0000000000000..2ababc7d96ea2 --- /dev/null +++ b/cpp/src/arrow/array/builder_union.h @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/array/builder_base.h" +#include "arrow/buffer-builder.h" + +namespace arrow { + +/// \class DenseUnionBuilder +/// +/// You need to call AppendChild for each of the children builders you want +/// to use. The function will return an int8_t, which is the type tag +/// associated with that child. You can then call Append with that tag +/// (followed by an append on the child builder) to add elements to +/// the union array. +/// +/// You can either specify the type when the UnionBuilder is constructed +/// or let the UnionBuilder infer the type at runtime (by omitting the +/// type argument from the constructor). +/// +/// This API is EXPERIMENTAL. +class ARROW_EXPORT DenseUnionBuilder : public ArrayBuilder { + public: + /// Use this constructor to incrementally build the union array along + /// with types, offsets, and null bitmap. + explicit DenseUnionBuilder(MemoryPool* pool, + const std::shared_ptr& type = NULLPTR); + + Status AppendNull() { + ARROW_RETURN_NOT_OK(types_builder_.Append(0)); + ARROW_RETURN_NOT_OK(offsets_builder_.Append(0)); + return AppendToBitmap(false); + } + + /// \brief Append an element to the UnionArray. This must be followed + /// by an append to the appropriate child builder. + /// \param[in] type index of the child the value will be appended + /// \param[in] offset offset of the value in that child + Status Append(int8_t type, int32_t offset) { + ARROW_RETURN_NOT_OK(types_builder_.Append(type)); + ARROW_RETURN_NOT_OK(offsets_builder_.Append(offset)); + return AppendToBitmap(true); + } + + Status FinishInternal(std::shared_ptr* out) override; + + /// \brief Make a new child builder available to the UnionArray + /// + /// \param[in] child the child builder + /// \param[in] field_name the name of the field in the union array type + /// if type inference is used + /// \return child index, which is the "type" argument that needs + /// to be passed to the "Append" method to add a new element to + /// the union array. + int8_t AppendChild(const std::shared_ptr& child, + const std::string& field_name = "") { + children_.push_back(child); + field_names_.push_back(field_name); + return static_cast(children_.size() - 1); + } + + private: + TypedBufferBuilder types_builder_; + TypedBufferBuilder offsets_builder_; + std::vector field_names_; +}; + +} // namespace arrow diff --git a/cpp/src/arrow/python/deserialize.cc b/cpp/src/arrow/python/deserialize.cc index f1a7eab8fcbda..f13070a5883f9 100644 --- a/cpp/src/arrow/python/deserialize.cc +++ b/cpp/src/arrow/python/deserialize.cc @@ -108,17 +108,16 @@ Status DeserializeArray(int32_t index, PyObject* base, const SerializedPyObject& return Status::OK(); } -Status GetValue(PyObject* context, const UnionArray& parent, const Array& arr, - int64_t index, int32_t type, PyObject* base, - const SerializedPyObject& blobs, PyObject** result) { - switch (arr.type()->id()) { - case Type::BOOL: +Status GetValue(PyObject* context, const Array& arr, int64_t index, int8_t type, + PyObject* base, const SerializedPyObject& blobs, PyObject** result) { + switch (type) { + case PythonType::BOOL: *result = PyBool_FromLong(checked_cast(arr).Value(index)); return Status::OK(); - case Type::INT64: { + case PythonType::PY2INT: + case PythonType::INT: { #if PY_MAJOR_VERSION < 3 - const std::string& child_name = parent.type()->child(type)->name(); - if (child_name == "py2_int") { + if (type == PythonType::PY2INT) { *result = PyInt_FromSsize_t(checked_cast(arr).Value(index)); return Status::OK(); } @@ -126,135 +125,151 @@ Status GetValue(PyObject* context, const UnionArray& parent, const Array& arr, *result = PyLong_FromSsize_t(checked_cast(arr).Value(index)); return Status::OK(); } - case Type::BINARY: { + case PythonType::BYTES: { auto view = checked_cast(arr).GetView(index); *result = PyBytes_FromStringAndSize(view.data(), view.length()); return CheckPyError(); } - case Type::STRING: { + case PythonType::STRING: { auto view = checked_cast(arr).GetView(index); *result = PyUnicode_FromStringAndSize(view.data(), view.length()); return CheckPyError(); } - case Type::HALF_FLOAT: { + case PythonType::HALF_FLOAT: { *result = PyHalf_FromHalf(checked_cast(arr).Value(index)); RETURN_IF_PYERROR(); return Status::OK(); } - case Type::FLOAT: + case PythonType::FLOAT: *result = PyFloat_FromDouble(checked_cast(arr).Value(index)); return Status::OK(); - case Type::DOUBLE: + case PythonType::DOUBLE: *result = PyFloat_FromDouble(checked_cast(arr).Value(index)); return Status::OK(); - case Type::DATE64: { + case PythonType::DATE64: { RETURN_NOT_OK(PyDateTime_from_int( checked_cast(arr).Value(index), TimeUnit::MICRO, result)); RETURN_IF_PYERROR(); return Status::OK(); } - case Type::STRUCT: { - const auto& s = checked_cast(arr); - const auto& l = checked_cast(*s.field(0)); - if (s.type()->child(0)->name() == "list") { - return DeserializeList(context, *l.values(), l.value_offset(index), - l.value_offset(index + 1), base, blobs, result); - } else if (s.type()->child(0)->name() == "tuple") { - return DeserializeTuple(context, *l.values(), l.value_offset(index), - l.value_offset(index + 1), base, blobs, result); - } else if (s.type()->child(0)->name() == "dict") { - return DeserializeDict(context, *l.values(), l.value_offset(index), - l.value_offset(index + 1), base, blobs, result); - } else if (s.type()->child(0)->name() == "set") { - return DeserializeSet(context, *l.values(), l.value_offset(index), + case PythonType::LIST: { + const auto& l = checked_cast(arr); + return DeserializeList(context, *l.values(), l.value_offset(index), + l.value_offset(index + 1), base, blobs, result); + } + case PythonType::DICT: { + const auto& l = checked_cast(arr); + return DeserializeDict(context, *l.values(), l.value_offset(index), + l.value_offset(index + 1), base, blobs, result); + } + case PythonType::TUPLE: { + const auto& l = checked_cast(arr); + return DeserializeTuple(context, *l.values(), l.value_offset(index), l.value_offset(index + 1), base, blobs, result); - } else { - DCHECK(false) << "unexpected StructArray type " << s.type()->child(0)->name(); - } } - default: { - const std::string& child_name = parent.type()->child(type)->name(); - if (child_name == "tensor") { - int32_t ref = checked_cast(arr).Value(index); - *result = wrap_tensor(blobs.tensors[ref]); - return Status::OK(); - } else if (child_name == "buffer") { - int32_t ref = checked_cast(arr).Value(index); - *result = wrap_buffer(blobs.buffers[ref]); - return Status::OK(); - } else if (child_name == "ndarray") { - int32_t ref = checked_cast(arr).Value(index); - return DeserializeArray(ref, base, blobs, result); - } else { - DCHECK(false) << "union tag " << type << " with child name '" << child_name - << "' not recognized"; - } + case PythonType::SET: { + const auto& l = checked_cast(arr); + return DeserializeSet(context, *l.values(), l.value_offset(index), + l.value_offset(index + 1), base, blobs, result); + } + case PythonType::TENSOR: { + int32_t ref = checked_cast(arr).Value(index); + *result = wrap_tensor(blobs.tensors[ref]); + return Status::OK(); + } + case PythonType::NDARRAY: { + int32_t ref = checked_cast(arr).Value(index); + return DeserializeArray(ref, base, blobs, result); + } + case PythonType::BUFFER: { + int32_t ref = checked_cast(arr).Value(index); + *result = wrap_buffer(blobs.buffers[ref]); + return Status::OK(); } + default: { ARROW_CHECK(false) << "union tag " << type << "' not recognized"; } } return Status::OK(); } -#define DESERIALIZE_SEQUENCE(CREATE_FN, SET_ITEM_FN) \ - const auto& data = checked_cast(array); \ - OwnedRef result(CREATE_FN(stop_idx - start_idx)); \ - const uint8_t* type_ids = data.raw_type_ids(); \ - const int32_t* value_offsets = data.raw_value_offsets(); \ - for (int64_t i = start_idx; i < stop_idx; ++i) { \ - if (data.IsNull(i)) { \ - Py_INCREF(Py_None); \ - SET_ITEM_FN(result.obj(), i - start_idx, Py_None); \ - } else { \ - int64_t offset = value_offsets[i]; \ - uint8_t type = type_ids[i]; \ - PyObject* value; \ - RETURN_NOT_OK(GetValue(context, data, *data.UnsafeChild(type), offset, type, base, \ - blobs, &value)); \ - SET_ITEM_FN(result.obj(), i - start_idx, value); \ - } \ - } \ - *out = result.detach(); \ - return Status::OK() - -Status DeserializeList(PyObject* context, const Array& array, int64_t start_idx, - int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs, - PyObject** out) { - DESERIALIZE_SEQUENCE(PyList_New, PyList_SET_ITEM); -} - -Status DeserializeTuple(PyObject* context, const Array& array, int64_t start_idx, - int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs, - PyObject** out) { - DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SET_ITEM); +std::vector GetPythonTypes(const UnionArray& data) { + std::vector result; + auto type = data.type(); + for (int i = 0; i < type->num_children(); ++i) { + // stoi is locale dependent, but should be ok for small integers + result.push_back(static_cast(std::stoi(type->child(i)->name()))); + } + return result; } -Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx, - int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs, - PyObject** out) { +template +Status DeserializeSequence(PyObject* context, const Array& array, int64_t start_idx, + int64_t stop_idx, PyObject* base, + const SerializedPyObject& blobs, + CreateSequenceFn&& create_sequence, SetItemFn&& set_item, + PyObject** out) { const auto& data = checked_cast(array); - OwnedRef result(PySet_New(nullptr)); + OwnedRef result(create_sequence(stop_idx - start_idx)); + RETURN_IF_PYERROR(); const uint8_t* type_ids = data.raw_type_ids(); const int32_t* value_offsets = data.raw_value_offsets(); + auto python_types = GetPythonTypes(data); for (int64_t i = start_idx; i < stop_idx; ++i) { if (data.IsNull(i)) { Py_INCREF(Py_None); - if (PySet_Add(result.obj(), Py_None) < 0) { - RETURN_IF_PYERROR(); - } + RETURN_NOT_OK(set_item(result.obj(), i - start_idx, Py_None)); } else { - int32_t offset = value_offsets[i]; - int8_t type = type_ids[i]; + int64_t offset = value_offsets[i]; + uint8_t type = type_ids[i]; PyObject* value; - RETURN_NOT_OK(GetValue(context, data, *data.UnsafeChild(type), offset, type, base, - blobs, &value)); - if (PySet_Add(result.obj(), value) < 0) { - RETURN_IF_PYERROR(); - } + RETURN_NOT_OK(GetValue(context, *data.UnsafeChild(type), offset, + python_types[type_ids[i]], base, blobs, &value)); + RETURN_NOT_OK(set_item(result.obj(), i - start_idx, value)); } } *out = result.detach(); return Status::OK(); } +Status DeserializeList(PyObject* context, const Array& array, int64_t start_idx, + int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs, + PyObject** out) { + return DeserializeSequence(context, array, start_idx, stop_idx, base, blobs, + [](int64_t size) { return PyList_New(size); }, + [](PyObject* seq, int64_t index, PyObject* item) { + PyList_SET_ITEM(seq, index, item); + return Status::OK(); + }, + out); +} + +Status DeserializeTuple(PyObject* context, const Array& array, int64_t start_idx, + int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs, + PyObject** out) { + return DeserializeSequence(context, array, start_idx, stop_idx, base, blobs, + [](int64_t size) { return PyTuple_New(size); }, + [](PyObject* seq, int64_t index, PyObject* item) { + PyTuple_SET_ITEM(seq, index, item); + return Status::OK(); + }, + out); +} + +Status DeserializeSet(PyObject* context, const Array& array, int64_t start_idx, + int64_t stop_idx, PyObject* base, const SerializedPyObject& blobs, + PyObject** out) { + return DeserializeSequence(context, array, start_idx, stop_idx, base, blobs, + [](int64_t size) { return PySet_New(nullptr); }, + [](PyObject* seq, int64_t index, PyObject* item) { + int err = PySet_Add(seq, item); + Py_DECREF(item); + if (err < 0) { + RETURN_IF_PYERROR(); + } + return Status::OK(); + }, + out); +} + Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out) { int64_t bytes_read; int32_t num_tensors; diff --git a/cpp/src/arrow/python/serialize.cc b/cpp/src/arrow/python/serialize.cc index ad2636af60c63..4dd4c04a6ccb5 100644 --- a/cpp/src/arrow/python/serialize.cc +++ b/cpp/src/arrow/python/serialize.cc @@ -29,6 +29,7 @@ #include #include "arrow/array.h" +#include "arrow/array/builder_union.h" #include "arrow/builder.h" #include "arrow/io/interfaces.h" #include "arrow/io/memory.h" @@ -55,6 +56,12 @@ using internal::checked_cast; namespace py { +class SequenceBuilder; +class DictBuilder; + +Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder, + int32_t recursion_depth, SerializedPyObject* blobs_out); + // A Sequence is a heterogeneous collections of elements. It can contain // scalar Python types, lists, tuples, dictionaries and tensors. class SequenceBuilder { @@ -63,241 +70,162 @@ class SequenceBuilder { : pool_(pool), types_(::arrow::int8(), pool), offsets_(::arrow::int32(), pool), - nones_(pool), - bools_(::arrow::boolean(), pool), - ints_(::arrow::int64(), pool), - py2_ints_(::arrow::int64(), pool), - bytes_(::arrow::binary(), pool), - strings_(pool), - half_floats_(::arrow::float16(), pool), - floats_(::arrow::float32(), pool), - doubles_(::arrow::float64(), pool), - date64s_(::arrow::date64(), pool), - tensor_indices_(::arrow::int32(), pool), - ndarray_indices_(::arrow::int32(), pool), - buffer_indices_(::arrow::int32(), pool), - list_offsets_({0}), - tuple_offsets_({0}), - dict_offsets_({0}), - set_offsets_({0}) {} + type_map_(PythonType::NUM_PYTHON_TYPES, -1) { + builder_.reset(new DenseUnionBuilder(pool)); + } // Appending a none to the sequence - Status AppendNone() { - RETURN_NOT_OK(offsets_.Append(0)); - RETURN_NOT_OK(types_.Append(0)); - return nones_.AppendNull(); - } + Status AppendNone() { return builder_->AppendNull(); } - Status Update(int64_t offset, int8_t* tag) { - if (*tag == -1) { - *tag = num_tags_++; - } + template + Status Update(BuilderType* child_builder, int8_t tag) { int32_t offset32 = -1; - RETURN_NOT_OK(internal::CastSize(offset, &offset32)); + RETURN_NOT_OK(internal::CastSize(child_builder->length(), &offset32)); DCHECK_GE(offset32, 0); - RETURN_NOT_OK(offsets_.Append(offset32)); - RETURN_NOT_OK(types_.Append(*tag)); - return nones_.Append(true); + return builder_->Append(tag, offset32); + } + + template + Status CreateAndUpdate(std::shared_ptr* child_builder, int8_t tag, + MakeBuilderFn make_builder) { + if (!*child_builder) { + child_builder->reset(make_builder()); + // std::to_string is locale dependent, but should be ok for small integers + type_map_[tag] = builder_->AppendChild(*child_builder, std::to_string(tag)); + } + return Update(child_builder->get(), type_map_[tag]); } template - Status AppendPrimitive(const T val, int8_t* tag, BuilderType* out) { - RETURN_NOT_OK(Update(out->length(), tag)); - return out->Append(val); + Status AppendPrimitive(std::shared_ptr* child_builder, const T val, + int8_t tag) { + RETURN_NOT_OK( + CreateAndUpdate(child_builder, tag, [this]() { return new BuilderType(pool_); })); + return (*child_builder)->Append(val); } // Appending a boolean to the sequence Status AppendBool(const bool data) { - return AppendPrimitive(data, &bool_tag_, &bools_); + return AppendPrimitive(&bools_, data, PythonType::BOOL); } // Appending a python 2 int64_t to the sequence Status AppendPy2Int64(const int64_t data) { - return AppendPrimitive(data, &py2_int_tag_, &py2_ints_); + return AppendPrimitive(&py2_ints_, data, PythonType::PY2INT); } // Appending an int64_t to the sequence Status AppendInt64(const int64_t data) { - return AppendPrimitive(data, &int_tag_, &ints_); + return AppendPrimitive(&ints_, data, PythonType::INT); } // Append a list of bytes to the sequence Status AppendBytes(const uint8_t* data, int32_t length) { - RETURN_NOT_OK(Update(bytes_.length(), &bytes_tag_)); - return bytes_.Append(data, length); + RETURN_NOT_OK(CreateAndUpdate(&bytes_, PythonType::BYTES, + [this]() { return new BinaryBuilder(pool_); })); + return bytes_->Append(data, length); } // Appending a string to the sequence Status AppendString(const char* data, int32_t length) { - RETURN_NOT_OK(Update(strings_.length(), &string_tag_)); - return strings_.Append(data, length); + RETURN_NOT_OK(CreateAndUpdate(&strings_, PythonType::STRING, + [this]() { return new StringBuilder(pool_); })); + return strings_->Append(data, length); } // Appending a half_float to the sequence Status AppendHalfFloat(const npy_half data) { - return AppendPrimitive(data, &half_float_tag_, &half_floats_); + return AppendPrimitive(&half_floats_, data, PythonType::HALF_FLOAT); } // Appending a float to the sequence Status AppendFloat(const float data) { - return AppendPrimitive(data, &float_tag_, &floats_); + return AppendPrimitive(&floats_, data, PythonType::FLOAT); } // Appending a double to the sequence Status AppendDouble(const double data) { - return AppendPrimitive(data, &double_tag_, &doubles_); + return AppendPrimitive(&doubles_, data, PythonType::DOUBLE); } // Appending a Date64 timestamp to the sequence Status AppendDate64(const int64_t timestamp) { - return AppendPrimitive(timestamp, &date64_tag_, &date64s_); + return AppendPrimitive(&date64s_, timestamp, PythonType::DATE64); } // Appending a tensor to the sequence // // \param tensor_index Index of the tensor in the object. Status AppendTensor(const int32_t tensor_index) { - RETURN_NOT_OK(Update(tensor_indices_.length(), &tensor_tag_)); - return tensor_indices_.Append(tensor_index); + RETURN_NOT_OK(CreateAndUpdate(&tensor_indices_, PythonType::TENSOR, + [this]() { return new Int32Builder(pool_); })); + return tensor_indices_->Append(tensor_index); } // Appending a numpy ndarray to the sequence // // \param tensor_index Index of the tensor in the object. Status AppendNdarray(const int32_t ndarray_index) { - RETURN_NOT_OK(Update(ndarray_indices_.length(), &ndarray_tag_)); - return ndarray_indices_.Append(ndarray_index); + RETURN_NOT_OK(CreateAndUpdate(&ndarray_indices_, PythonType::NDARRAY, + [this]() { return new Int32Builder(pool_); })); + return ndarray_indices_->Append(ndarray_index); } // Appending a buffer to the sequence // // \param buffer_index Indes of the buffer in the object. Status AppendBuffer(const int32_t buffer_index) { - RETURN_NOT_OK(Update(buffer_indices_.length(), &buffer_tag_)); - return buffer_indices_.Append(buffer_index); - } - - // Add a sublist to the sequence. The data contained in the sublist will be - // specified in the "Finish" method. - // - // To construct l = [[11, 22], 33, [44, 55]] you would for example run - // list = ListBuilder(); - // list.AppendList(2); - // list.Append(33); - // list.AppendList(2); - // list.Finish([11, 22, 44, 55]); - // list.Finish(); - - // \param size - // The size of the sublist - Status AppendList(Py_ssize_t size) { - int32_t offset; - RETURN_NOT_OK(internal::CastSize(list_offsets_.back() + size, &offset)); - RETURN_NOT_OK(Update(list_offsets_.size() - 1, &list_tag_)); - list_offsets_.push_back(offset); - return Status::OK(); + RETURN_NOT_OK(CreateAndUpdate(&buffer_indices_, PythonType::BUFFER, + [this]() { return new Int32Builder(pool_); })); + return buffer_indices_->Append(buffer_index); + } + + Status AppendSequence(PyObject* context, PyObject* sequence, int8_t tag, + std::shared_ptr& target_sequence, + std::unique_ptr& values, int32_t recursion_depth, + SerializedPyObject* blobs_out) { + if (recursion_depth >= kMaxRecursionDepth) { + return Status::NotImplemented( + "This object exceeds the maximum recursion depth. It may contain itself " + "recursively."); + } + RETURN_NOT_OK(CreateAndUpdate(&target_sequence, tag, [this, &values]() { + values.reset(new SequenceBuilder(pool_)); + return new ListBuilder(pool_, values->builder()); + })); + RETURN_NOT_OK(target_sequence->Append()); + return internal::VisitIterable( + sequence, [&](PyObject* obj, bool* keep_going /* unused */) { + return Append(context, obj, values.get(), recursion_depth, blobs_out); + }); } - Status AppendTuple(Py_ssize_t size) { - int32_t offset; - RETURN_NOT_OK(internal::CastSize(tuple_offsets_.back() + size, &offset)); - RETURN_NOT_OK(Update(tuple_offsets_.size() - 1, &tuple_tag_)); - tuple_offsets_.push_back(offset); - return Status::OK(); + Status AppendList(PyObject* context, PyObject* list, int32_t recursion_depth, + SerializedPyObject* blobs_out) { + return AppendSequence(context, list, PythonType::LIST, lists_, list_values_, + recursion_depth, blobs_out); } - Status AppendDict(Py_ssize_t size) { - int32_t offset; - RETURN_NOT_OK(internal::CastSize(dict_offsets_.back() + size, &offset)); - RETURN_NOT_OK(Update(dict_offsets_.size() - 1, &dict_tag_)); - dict_offsets_.push_back(offset); - return Status::OK(); - } - - Status AppendSet(Py_ssize_t size) { - int32_t offset; - RETURN_NOT_OK(internal::CastSize(set_offsets_.back() + size, &offset)); - RETURN_NOT_OK(Update(set_offsets_.size() - 1, &set_tag_)); - set_offsets_.push_back(offset); - return Status::OK(); + Status AppendTuple(PyObject* context, PyObject* tuple, int32_t recursion_depth, + SerializedPyObject* blobs_out) { + return AppendSequence(context, tuple, PythonType::TUPLE, tuples_, tuple_values_, + recursion_depth, blobs_out); } - template - Status AddElement(const int8_t tag, BuilderType* out, const std::string& name = "") { - if (tag != -1) { - fields_[tag] = ::arrow::field(name, out->type()); - RETURN_NOT_OK(out->Finish(&children_[tag])); - RETURN_NOT_OK(nones_.Append(true)); - type_ids_.push_back(tag); - } - return Status::OK(); + Status AppendSet(PyObject* context, PyObject* set, int32_t recursion_depth, + SerializedPyObject* blobs_out) { + return AppendSequence(context, set, PythonType::SET, sets_, set_values_, + recursion_depth, blobs_out); } - Status AddSubsequence(int8_t tag, const Array* data, - const std::vector& offsets, const std::string& name) { - if (data != nullptr) { - DCHECK(data->length() == offsets.back()); - std::shared_ptr offset_array; - Int32Builder builder(::arrow::int32(), pool_); - RETURN_NOT_OK(builder.AppendValues(offsets.data(), offsets.size())); - RETURN_NOT_OK(builder.Finish(&offset_array)); - std::shared_ptr list_array; - RETURN_NOT_OK(ListArray::FromArrays(*offset_array, *data, pool_, &list_array)); - auto field = ::arrow::field(name, list_array->type()); - auto type = ::arrow::struct_({field}); - fields_[tag] = ::arrow::field("", type); - children_[tag] = std::shared_ptr( - new StructArray(type, list_array->length(), {list_array})); - RETURN_NOT_OK(nones_.Append(true)); - type_ids_.push_back(tag); - } else { - DCHECK_EQ(offsets.size(), 1); - } - return Status::OK(); - } + Status AppendDict(PyObject* context, PyObject* dict, int32_t recursion_depth, + SerializedPyObject* blobs_out); // Finish building the sequence and return the result. // Input arrays may be nullptr - Status Finish(const Array* list_data, const Array* tuple_data, const Array* dict_data, - const Array* set_data, std::shared_ptr* out) { - fields_.resize(num_tags_); - children_.resize(num_tags_); - - RETURN_NOT_OK(AddElement(bool_tag_, &bools_)); - RETURN_NOT_OK(AddElement(int_tag_, &ints_)); - RETURN_NOT_OK(AddElement(py2_int_tag_, &py2_ints_, "py2_int")); - RETURN_NOT_OK(AddElement(string_tag_, &strings_)); - RETURN_NOT_OK(AddElement(bytes_tag_, &bytes_)); - RETURN_NOT_OK(AddElement(half_float_tag_, &half_floats_)); - RETURN_NOT_OK(AddElement(float_tag_, &floats_)); - RETURN_NOT_OK(AddElement(double_tag_, &doubles_)); - RETURN_NOT_OK(AddElement(date64_tag_, &date64s_)); - RETURN_NOT_OK(AddElement(tensor_tag_, &tensor_indices_, "tensor")); - RETURN_NOT_OK(AddElement(buffer_tag_, &buffer_indices_, "buffer")); - RETURN_NOT_OK(AddElement(ndarray_tag_, &ndarray_indices_, "ndarray")); - - RETURN_NOT_OK(AddSubsequence(list_tag_, list_data, list_offsets_, "list")); - RETURN_NOT_OK(AddSubsequence(tuple_tag_, tuple_data, tuple_offsets_, "tuple")); - RETURN_NOT_OK(AddSubsequence(dict_tag_, dict_data, dict_offsets_, "dict")); - RETURN_NOT_OK(AddSubsequence(set_tag_, set_data, set_offsets_, "set")); - - std::shared_ptr types_array; - RETURN_NOT_OK(types_.Finish(&types_array)); - const auto& types = checked_cast(*types_array); - - std::shared_ptr offsets_array; - RETURN_NOT_OK(offsets_.Finish(&offsets_array)); - const auto& offsets = checked_cast(*offsets_array); - - std::shared_ptr nones_array; - RETURN_NOT_OK(nones_.Finish(&nones_array)); - const auto& nones = checked_cast(*nones_array); - - auto type = ::arrow::union_(fields_, type_ids_, UnionMode::DENSE); - out->reset(new UnionArray(type, types.length(), children_, types.values(), - offsets.values(), nones.null_bitmap(), nones.null_count())); - return Status::OK(); - } + Status Finish(std::shared_ptr* out) { return builder_->Finish(out); } + + std::shared_ptr builder() { return builder_; } private: MemoryPool* pool_; @@ -305,55 +233,33 @@ class SequenceBuilder { Int8Builder types_; Int32Builder offsets_; - BooleanBuilder nones_; - BooleanBuilder bools_; - Int64Builder ints_; - Int64Builder py2_ints_; - BinaryBuilder bytes_; - StringBuilder strings_; - HalfFloatBuilder half_floats_; - FloatBuilder floats_; - DoubleBuilder doubles_; - Date64Builder date64s_; - - Int32Builder tensor_indices_; - Int32Builder ndarray_indices_; - Int32Builder buffer_indices_; - - std::vector list_offsets_; - std::vector tuple_offsets_; - std::vector dict_offsets_; - std::vector set_offsets_; - - // Tags for members of the sequence. If they are set to -1 it means - // they are not used and will not part be of the metadata when we call - // SequenceBuilder::Finish. If a member with one of the tags is added, - // the associated variable gets a unique index starting from 0. This - // happens in the UPDATE macro in sequence.cc. - int8_t bool_tag_ = -1; - int8_t int_tag_ = -1; - int8_t py2_int_tag_ = -1; - int8_t string_tag_ = -1; - int8_t bytes_tag_ = -1; - int8_t half_float_tag_ = -1; - int8_t float_tag_ = -1; - int8_t double_tag_ = -1; - int8_t date64_tag_ = -1; - - int8_t tensor_tag_ = -1; - int8_t buffer_tag_ = -1; - int8_t ndarray_tag_ = -1; - int8_t list_tag_ = -1; - int8_t tuple_tag_ = -1; - int8_t dict_tag_ = -1; - int8_t set_tag_ = -1; - - int8_t num_tags_ = 0; - - // Members for the output union constructed in Finish - std::vector> fields_; - std::vector> children_; - std::vector type_ids_; + /// Mapping from PythonType to child index + std::vector type_map_; + + std::shared_ptr bools_; + std::shared_ptr ints_; + std::shared_ptr py2_ints_; + std::shared_ptr bytes_; + std::shared_ptr strings_; + std::shared_ptr half_floats_; + std::shared_ptr floats_; + std::shared_ptr doubles_; + std::shared_ptr date64s_; + + std::unique_ptr list_values_; + std::shared_ptr lists_; + std::unique_ptr dict_values_; + std::shared_ptr dicts_; + std::unique_ptr tuple_values_; + std::shared_ptr tuples_; + std::unique_ptr set_values_; + std::shared_ptr sets_; + + std::shared_ptr tensor_indices_; + std::shared_ptr ndarray_indices_; + std::shared_ptr buffer_indices_; + + std::shared_ptr builder_; }; // Constructing dictionaries of key/value pairs. Sequences of @@ -362,7 +268,9 @@ class SequenceBuilder { // can be obtained via the Finish method. class DictBuilder { public: - explicit DictBuilder(MemoryPool* pool = nullptr) : keys_(pool), vals_(pool) {} + explicit DictBuilder(MemoryPool* pool = nullptr) : keys_(pool), vals_(pool) { + builder_.reset(new StructBuilder(nullptr, pool, {keys_.builder(), vals_.builder()})); + } // Builder for the keys of the dictionary SequenceBuilder& keys() { return keys_; } @@ -371,38 +279,55 @@ class DictBuilder { // Construct an Arrow StructArray representing the dictionary. // Contains a field "keys" for the keys and "vals" for the values. - // \param val_list_data - // List containing the data from nested lists in the value - // list of the dictionary - // - // \param val_dict_data - // List containing the data from nested dictionaries in the - // value list of the dictionary - Status Finish(const Array* key_tuple_data, const Array* key_dict_data, - const Array* val_list_data, const Array* val_tuple_data, - const Array* val_dict_data, const Array* val_set_data, - std::shared_ptr* out) { - // lists and sets can't be keys of dicts in Python, that is why for - // the keys we do not need to collect sublists - std::shared_ptr keys, vals; - RETURN_NOT_OK(keys_.Finish(nullptr, key_tuple_data, key_dict_data, nullptr, &keys)); - RETURN_NOT_OK( - vals_.Finish(val_list_data, val_tuple_data, val_dict_data, val_set_data, &vals)); - auto keys_field = std::make_shared("keys", keys->type()); - auto vals_field = std::make_shared("vals", vals->type()); - auto type = std::make_shared( - std::vector>({keys_field, vals_field})); - std::vector> field_arrays({keys, vals}); - DCHECK(keys->length() == vals->length()); - out->reset(new StructArray(type, keys->length(), field_arrays)); - return Status::OK(); - } + Status Finish(std::shared_ptr* out) { return builder_->Finish(out); } + + std::shared_ptr builder() { return builder_; } private: SequenceBuilder keys_; SequenceBuilder vals_; + std::shared_ptr builder_; }; +Status SequenceBuilder::AppendDict(PyObject* context, PyObject* dict, + int32_t recursion_depth, + SerializedPyObject* blobs_out) { + if (recursion_depth >= kMaxRecursionDepth) { + return Status::NotImplemented( + "This object exceeds the maximum recursion depth. It may contain itself " + "recursively."); + } + RETURN_NOT_OK(CreateAndUpdate(&dicts_, PythonType::DICT, [this]() { + dict_values_.reset(new DictBuilder(pool_)); + return new ListBuilder(pool_, dict_values_->builder()); + })); + RETURN_NOT_OK(dicts_->Append()); + PyObject* key; + PyObject* value; + Py_ssize_t pos = 0; + while (PyDict_Next(dict, &pos, &key, &value)) { + RETURN_NOT_OK(dict_values_->builder()->Append()); + RETURN_NOT_OK( + Append(context, key, &dict_values_->keys(), recursion_depth + 1, blobs_out)); + RETURN_NOT_OK( + Append(context, value, &dict_values_->vals(), recursion_depth + 1, blobs_out)); + } + + // This block is used to decrement the reference counts of the results + // returned by the serialization callback, which is called in AppendArray, + // in DeserializeDict and in Append + static PyObject* py_type = PyUnicode_FromString("_pytype_"); + if (PyDict_Contains(dict, py_type)) { + // If the dictionary contains the key "_pytype_", then the user has to + // have registered a callback. + if (context == Py_None) { + return Status::Invalid("No serialization callback set"); + } + Py_XDECREF(dict); + } + return Status::OK(); +} + Status CallCustomCallback(PyObject* context, PyObject* method_name, PyObject* elem, PyObject** result) { *result = NULL; @@ -433,16 +358,8 @@ Status CallDeserializeCallback(PyObject* context, PyObject* value, return CallCustomCallback(context, method_name.obj(), value, deserialized_object); } -Status SerializeDict(PyObject* context, std::vector dicts, - int32_t recursion_depth, std::shared_ptr* out, - SerializedPyObject* blobs_out); - -Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder, - std::vector* subdicts, SerializedPyObject* blobs_out); - -Status SerializeSequences(PyObject* context, std::vector sequences, - int32_t recursion_depth, std::shared_ptr* out, - SerializedPyObject* blobs_out); +Status AppendArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder, + int32_t recursion_depth, SerializedPyObject* blobs_out); template Status AppendIntegerScalar(PyObject* obj, SequenceBuilder* builder) { @@ -502,9 +419,7 @@ Status AppendScalar(PyObject* obj, SequenceBuilder* builder) { } Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder, - std::vector* sublists, std::vector* subtuples, - std::vector* subdicts, std::vector* subsets, - SerializedPyObject* blobs_out) { + int32_t recursion_depth, SerializedPyObject* blobs_out) { // The bool case must precede the int case (PyInt_Check passes for bools) if (PyBool_Check(elem)) { RETURN_NOT_OK(builder->AppendBool(elem == Py_True)); @@ -523,8 +438,8 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder, PyObject* serialized_object; // The reference count of serialized_object will be decremented in SerializeDict RETURN_NOT_OK(CallSerializeCallback(context, elem, &serialized_object)); - RETURN_NOT_OK(builder->AppendDict(PyDict_Size(serialized_object))); - subdicts->push_back(serialized_object); + RETURN_NOT_OK( + builder->AppendDict(context, serialized_object, recursion_depth, blobs_out)); } #if PY_MAJOR_VERSION < 3 } else if (PyInt_Check(elem)) { @@ -542,22 +457,18 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder, RETURN_NOT_OK(internal::CastSize(view.size, &size)); RETURN_NOT_OK(builder->AppendString(view.bytes, size)); } else if (PyList_CheckExact(elem)) { - RETURN_NOT_OK(builder->AppendList(PyList_Size(elem))); - sublists->push_back(elem); + RETURN_NOT_OK(builder->AppendList(context, elem, recursion_depth, blobs_out)); } else if (PyDict_CheckExact(elem)) { - RETURN_NOT_OK(builder->AppendDict(PyDict_Size(elem))); - subdicts->push_back(elem); + RETURN_NOT_OK(builder->AppendDict(context, elem, recursion_depth, blobs_out)); } else if (PyTuple_CheckExact(elem)) { - RETURN_NOT_OK(builder->AppendTuple(PyTuple_Size(elem))); - subtuples->push_back(elem); + RETURN_NOT_OK(builder->AppendTuple(context, elem, recursion_depth, blobs_out)); } else if (PySet_Check(elem)) { - RETURN_NOT_OK(builder->AppendSet(PySet_Size(elem))); - subsets->push_back(elem); + RETURN_NOT_OK(builder->AppendSet(context, elem, recursion_depth, blobs_out)); } else if (PyArray_IsScalar(elem, Generic)) { RETURN_NOT_OK(AppendScalar(elem, builder)); } else if (PyArray_CheckExact(elem)) { - RETURN_NOT_OK(SerializeArray(context, reinterpret_cast(elem), builder, - subdicts, blobs_out)); + RETURN_NOT_OK(AppendArray(context, reinterpret_cast(elem), builder, + recursion_depth, blobs_out)); } else if (elem == Py_None) { RETURN_NOT_OK(builder->AppendNone()); } else if (PyDateTime_Check(elem)) { @@ -578,14 +489,14 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder, PyObject* serialized_object; // The reference count of serialized_object will be decremented in SerializeDict RETURN_NOT_OK(CallSerializeCallback(context, elem, &serialized_object)); - RETURN_NOT_OK(builder->AppendDict(PyDict_Size(serialized_object))); - subdicts->push_back(serialized_object); + RETURN_NOT_OK( + builder->AppendDict(context, serialized_object, recursion_depth, blobs_out)); } return Status::OK(); } -Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder, - std::vector* subdicts, SerializedPyObject* blobs_out) { +Status AppendArray(PyObject* context, PyArrayObject* array, SequenceBuilder* builder, + int32_t recursion_depth, SerializedPyObject* blobs_out) { int dtype = PyArray_TYPE(array); switch (dtype) { case NPY_UINT8: @@ -611,126 +522,10 @@ Status SerializeArray(PyObject* context, PyArrayObject* array, SequenceBuilder* // The reference count of serialized_object will be decremented in SerializeDict RETURN_NOT_OK(CallSerializeCallback(context, reinterpret_cast(array), &serialized_object)); - RETURN_NOT_OK(builder->AppendDict(PyDict_Size(serialized_object))); - subdicts->push_back(serialized_object); - } - } - return Status::OK(); -} - -Status SerializeSequences(PyObject* context, std::vector sequences, - int32_t recursion_depth, std::shared_ptr* out, - SerializedPyObject* blobs_out) { - DCHECK(out); - if (recursion_depth >= kMaxRecursionDepth) { - return Status::NotImplemented( - "This object exceeds the maximum recursion depth. It may contain itself " - "recursively."); - } - SequenceBuilder builder; - std::vector sublists, subtuples, subdicts, subsets; - for (const auto& sequence : sequences) { - RETURN_NOT_OK(internal::VisitIterable( - sequence, [&](PyObject* obj, bool* keep_going /* unused */) { - return Append(context, obj, &builder, &sublists, &subtuples, &subdicts, - &subsets, blobs_out); - })); - } - std::shared_ptr list; - if (sublists.size() > 0) { - RETURN_NOT_OK( - SerializeSequences(context, sublists, recursion_depth + 1, &list, blobs_out)); - } - std::shared_ptr tuple; - if (subtuples.size() > 0) { - RETURN_NOT_OK( - SerializeSequences(context, subtuples, recursion_depth + 1, &tuple, blobs_out)); - } - std::shared_ptr dict; - if (subdicts.size() > 0) { - RETURN_NOT_OK( - SerializeDict(context, subdicts, recursion_depth + 1, &dict, blobs_out)); - } - std::shared_ptr set; - if (subsets.size() > 0) { - RETURN_NOT_OK( - SerializeSequences(context, subsets, recursion_depth + 1, &set, blobs_out)); - } - return builder.Finish(list.get(), tuple.get(), dict.get(), set.get(), out); -} - -Status SerializeDict(PyObject* context, std::vector dicts, - int32_t recursion_depth, std::shared_ptr* out, - SerializedPyObject* blobs_out) { - DictBuilder result; - if (recursion_depth >= kMaxRecursionDepth) { - return Status::NotImplemented( - "This object exceeds the maximum recursion depth. It may contain itself " - "recursively."); - } - std::vector key_tuples, key_dicts, val_lists, val_tuples, val_dicts, - val_sets, dummy; - for (const auto& dict : dicts) { - PyObject* key; - PyObject* value; - Py_ssize_t pos = 0; - while (PyDict_Next(dict, &pos, &key, &value)) { - RETURN_NOT_OK(Append(context, key, &result.keys(), &dummy, &key_tuples, &key_dicts, - &dummy, blobs_out)); - DCHECK_EQ(dummy.size(), 0); - RETURN_NOT_OK(Append(context, value, &result.vals(), &val_lists, &val_tuples, - &val_dicts, &val_sets, blobs_out)); - } - } - std::shared_ptr key_tuples_arr; - if (key_tuples.size() > 0) { - RETURN_NOT_OK(SerializeSequences(context, key_tuples, recursion_depth + 1, - &key_tuples_arr, blobs_out)); - } - std::shared_ptr key_dicts_arr; - if (key_dicts.size() > 0) { - RETURN_NOT_OK(SerializeDict(context, key_dicts, recursion_depth + 1, &key_dicts_arr, - blobs_out)); - } - std::shared_ptr val_list_arr; - if (val_lists.size() > 0) { - RETURN_NOT_OK(SerializeSequences(context, val_lists, recursion_depth + 1, - &val_list_arr, blobs_out)); - } - std::shared_ptr val_tuples_arr; - if (val_tuples.size() > 0) { - RETURN_NOT_OK(SerializeSequences(context, val_tuples, recursion_depth + 1, - &val_tuples_arr, blobs_out)); - } - std::shared_ptr val_dict_arr; - if (val_dicts.size() > 0) { - RETURN_NOT_OK( - SerializeDict(context, val_dicts, recursion_depth + 1, &val_dict_arr, blobs_out)); - } - std::shared_ptr val_set_arr; - if (val_sets.size() > 0) { - RETURN_NOT_OK(SerializeSequences(context, val_sets, recursion_depth + 1, &val_set_arr, - blobs_out)); - } - RETURN_NOT_OK(result.Finish(key_tuples_arr.get(), key_dicts_arr.get(), - val_list_arr.get(), val_tuples_arr.get(), - val_dict_arr.get(), val_set_arr.get(), out)); - - // This block is used to decrement the reference counts of the results - // returned by the serialization callback, which is called in SerializeArray, - // in DeserializeDict and in Append - static PyObject* py_type = PyUnicode_FromString("_pytype_"); - for (const auto& dict : dicts) { - if (PyDict_Contains(dict, py_type)) { - // If the dictionary contains the key "_pytype_", then the user has to - // have registered a callback. - if (context == Py_None) { - return Status::Invalid("No serialization callback set"); - } - Py_XDECREF(dict); + RETURN_NOT_OK(builder->AppendDict(context, serialized_object, recursion_depth + 1, + blobs_out)); } } - return Status::OK(); } @@ -744,9 +539,13 @@ Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject PyAcquireGIL lock; PyDateTime_IMPORT; import_pyarrow(); - std::vector sequences = {sequence}; + SequenceBuilder builder; + RETURN_NOT_OK(internal::VisitIterable( + sequence, [&](PyObject* obj, bool* keep_going /* unused */) { + return Append(context, obj, &builder, 0, out); + })); std::shared_ptr array; - RETURN_NOT_OK(SerializeSequences(context, sequences, 0, &array, out)); + RETURN_NOT_OK(builder.Finish(&array)); out->batch = MakeBatch(array); return Status::OK(); } @@ -756,7 +555,7 @@ Status SerializeNdarray(std::shared_ptr tensor, SerializedPyObject* out) SequenceBuilder builder; RETURN_NOT_OK(builder.AppendNdarray(static_cast(out->ndarrays.size()))); out->ndarrays.push_back(tensor); - RETURN_NOT_OK(builder.Finish(nullptr, nullptr, nullptr, nullptr, &array)); + RETURN_NOT_OK(builder.Finish(&array)); out->batch = MakeBatch(array); return Status::OK(); } diff --git a/cpp/src/arrow/python/serialize.h b/cpp/src/arrow/python/serialize.h index 9a9cc65087d55..6cdbbe5053f04 100644 --- a/cpp/src/arrow/python/serialize.h +++ b/cpp/src/arrow/python/serialize.h @@ -107,6 +107,28 @@ Status WriteNdarrayHeader(std::shared_ptr dtype, const std::vector& shape, int64_t tensor_num_bytes, io::OutputStream* dst); +struct PythonType { + enum type { + BOOL, + INT, + PY2INT, + BYTES, + STRING, + HALF_FLOAT, + FLOAT, + DOUBLE, + DATE64, + LIST, + DICT, + TUPLE, + SET, + TENSOR, + NDARRAY, + BUFFER, + NUM_PYTHON_TYPES + }; +}; + } // namespace py } // namespace arrow From 641c6990cdc702c736921c78448cf3a175c97863 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 31 Jan 2019 14:18:39 +0100 Subject: [PATCH 09/20] ARROW-4198: [Gandiva] Added support to cast timestamp Added howard hinnant date project as a third party library. Used system timezone database for timezone information. Author: Antoine Pitrou Author: shyam Closes #3352 from shyambits2004/timestamp and squashes the following commits: 882a5cf6 Tweak wording of vendored date library README 7f524805 Small tweaks to license wording for the date library 9ee8eff4 ARROW-4198 : Added support to cast timestamp --- LICENSE.txt | 6 +- cpp/src/arrow/CMakeLists.txt | 2 + cpp/src/arrow/util/parsing.h | 13 +- cpp/src/arrow/vendored/date.h | 6540 -------------- cpp/src/arrow/vendored/datetime/README.md | 21 + cpp/src/arrow/vendored/datetime/date.h | 8028 +++++++++++++++++ cpp/src/arrow/vendored/datetime/ios.h | 56 + cpp/src/arrow/vendored/datetime/tz.cpp | 3794 ++++++++ cpp/src/arrow/vendored/datetime/tz.h | 2593 ++++++ cpp/src/arrow/vendored/datetime/tz_private.h | 324 + cpp/src/gandiva/CMakeLists.txt | 2 + cpp/src/gandiva/cast_time.cc | 82 + cpp/src/gandiva/exported_funcs.h | 6 + cpp/src/gandiva/function_registry.cc | 1 - cpp/src/gandiva/function_registry_datetime.cc | 4 + cpp/src/gandiva/gdv_function_stubs.h | 3 + cpp/src/gandiva/precompiled/CMakeLists.txt | 3 +- .../gandiva/precompiled/epoch_time_point.h | 37 +- cpp/src/gandiva/precompiled/time.cc | 131 +- cpp/src/gandiva/precompiled/time_fields.h | 37 + cpp/src/gandiva/precompiled/time_test.cc | 52 +- cpp/src/gandiva/precompiled/types.h | 2 + cpp/src/gandiva/to_date_holder.cc | 7 +- 23 files changed, 15165 insertions(+), 6579 deletions(-) delete mode 100644 cpp/src/arrow/vendored/date.h create mode 100644 cpp/src/arrow/vendored/datetime/README.md create mode 100644 cpp/src/arrow/vendored/datetime/date.h create mode 100644 cpp/src/arrow/vendored/datetime/ios.h create mode 100644 cpp/src/arrow/vendored/datetime/tz.cpp create mode 100644 cpp/src/arrow/vendored/datetime/tz.h create mode 100644 cpp/src/arrow/vendored/datetime/tz_private.h create mode 100644 cpp/src/gandiva/cast_time.cc create mode 100644 cpp/src/gandiva/precompiled/time_fields.h diff --git a/LICENSE.txt b/LICENSE.txt index ad2255d431066..4bb80b93de459 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -681,7 +681,11 @@ See the License for the specific language governing permissions and limitations under the License. -------------------------------------------------------------------------------- -The file cpp/src/arrow/vendored/date.h has the following license (MIT) +The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, +cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, +cpp/src/arrow/vendored/datetime/tz.cpp are adapted from +Howard Hinnant's date library (https://github.com/HowardHinnant/date) +It is licensed under MIT license. The MIT License (MIT) Copyright (c) 2015, 2016, 2017 Howard Hinnant diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 838e3966ee895..1dba5898c0a7a 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -115,6 +115,8 @@ set(ARROW_SRCS util/thread-pool.cc util/trie.cc util/utf8.cc + + vendored/datetime/tz.cpp ) if ("${COMPILER_FAMILY}" STREQUAL "clang") diff --git a/cpp/src/arrow/util/parsing.h b/cpp/src/arrow/util/parsing.h index 23e7061ac8738..0d8eb97f873f6 100644 --- a/cpp/src/arrow/util/parsing.h +++ b/cpp/src/arrow/util/parsing.h @@ -34,7 +34,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/checked_cast.h" -#include "arrow/vendored/date.h" +#include "arrow/vendored/datetime/date.h" namespace arrow { namespace internal { @@ -375,7 +375,7 @@ class StringConverter { // - "YYYY-MM-DD[ T]hh:mm:ss" // - "YYYY-MM-DD[ T]hh:mm:ssZ" // UTC is always assumed, and the DataType's timezone is ignored. - date::year_month_day ymd; + arrow::util::date::year_month_day ymd; if (ARROW_PREDICT_FALSE(length < 10)) { return false; } @@ -383,7 +383,7 @@ class StringConverter { if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) { return false; } - return ConvertTimePoint(date::sys_days(ymd), out); + return ConvertTimePoint(arrow::util::date::sys_days(ymd), out); } if (ARROW_PREDICT_FALSE(s[10] != ' ') && ARROW_PREDICT_FALSE(s[10] != 'T')) { return false; @@ -399,7 +399,7 @@ class StringConverter { if (ARROW_PREDICT_FALSE(!ParseHH_MM_SS(s + 11, &seconds))) { return false; } - return ConvertTimePoint(date::sys_days(ymd) + seconds, out); + return ConvertTimePoint(arrow::util::date::sys_days(ymd) + seconds, out); } return false; } @@ -428,7 +428,7 @@ class StringConverter { return true; } - bool ParseYYYY_MM_DD(const char* s, date::year_month_day* out) { + bool ParseYYYY_MM_DD(const char* s, arrow::util::date::year_month_day* out) { uint16_t year; uint8_t month, day; if (ARROW_PREDICT_FALSE(s[4] != '-') || ARROW_PREDICT_FALSE(s[7] != '-')) { @@ -443,7 +443,8 @@ class StringConverter { if (ARROW_PREDICT_FALSE(!detail::ParseUnsigned(s + 8, 2, &day))) { return false; } - *out = {date::year{year}, date::month{month}, date::day{day}}; + *out = {arrow::util::date::year{year}, arrow::util::date::month{month}, + arrow::util::date::day{day}}; return out->ok(); } diff --git a/cpp/src/arrow/vendored/date.h b/cpp/src/arrow/vendored/date.h deleted file mode 100644 index aa7648899b902..0000000000000 --- a/cpp/src/arrow/vendored/date.h +++ /dev/null @@ -1,6540 +0,0 @@ -// Vendored from https://github.com/HowardHinnant/date/ - -#ifndef DATE_H -#define DATE_H - -// The MIT License (MIT) -// -// Copyright (c) 2015, 2016, 2017 Howard Hinnant -// Copyright (c) 2016 Adrian Colomitchi -// Copyright (c) 2017 Florian Dang -// Copyright (c) 2017 Paul Thompson -// Copyright (c) 2018 Tomasz Kamiński -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -// -// Our apologies. When the previous paragraph was written, lowercase had not yet -// been invented (that would involve another several millennia of evolution). -// We did not mean to shout. - -#ifndef HAS_STRING_VIEW -#if __cplusplus >= 201703 -#define HAS_STRING_VIEW 1 -#else -#define HAS_STRING_VIEW 0 -#endif -#endif // HAS_STRING_VIEW - -#include -#include -#include -#include -#include -#if !(__cplusplus >= 201402) -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAS_STRING_VIEW -#include -#endif -#include -#include - -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#if __GNUC__ < 5 -// GCC 4.9 Bug 61489 Wrong warning with -Wmissing-field-initializers -#pragma GCC diagnostic ignored "-Wmissing-field-initializers" -#endif -#endif - -namespace date { - -//---------------+ -// Configuration | -//---------------+ - -#ifndef ONLY_C_LOCALE -#define ONLY_C_LOCALE 0 -#endif - -#if defined(_MSC_VER) && (!defined(__clang__) || (_MSC_VER < 1910)) -// MSVC -#define _SILENCE_CXX17_UNCAUGHT_EXCEPTION_DEPRECATION_WARNING -#if _MSC_VER < 1910 -// before VS2017 -#define CONSTDATA const -#define CONSTCD11 -#define CONSTCD14 -#define NOEXCEPT _NOEXCEPT -#else -// VS2017 and later -#define CONSTDATA constexpr const -#define CONSTCD11 constexpr -#define CONSTCD14 constexpr -#define NOEXCEPT noexcept -#endif - -#elif defined(__SUNPRO_CC) && __SUNPRO_CC <= 0x5150 -// Oracle Developer Studio 12.6 and earlier -#define CONSTDATA constexpr const -#define CONSTCD11 constexpr -#define CONSTCD14 -#define NOEXCEPT noexcept - -#elif __cplusplus >= 201402 -// C++14 -#define CONSTDATA constexpr const -#define CONSTCD11 constexpr -#define CONSTCD14 constexpr -#define NOEXCEPT noexcept -#else -// C++11 -#define CONSTDATA constexpr const -#define CONSTCD11 constexpr -#define CONSTCD14 -#define NOEXCEPT noexcept -#endif - -#ifndef HAS_VOID_T -#if __cplusplus >= 201703 -#define HAS_VOID_T 1 -#else -#define HAS_VOID_T 0 -#endif -#endif // HAS_VOID_T - -// Protect from Oracle sun macro -#ifdef sun -#undef sun -#endif - -//-----------+ -// Interface | -//-----------+ - -// durations - -using days = std::chrono::duration< - int, std::ratio_multiply, std::chrono::hours::period>>; - -using weeks = - std::chrono::duration, days::period>>; - -using years = - std::chrono::duration, days::period>>; - -using months = - std::chrono::duration>>; - -// time_point - -template -using sys_time = std::chrono::time_point; - -using sys_days = sys_time; -using sys_seconds = sys_time; - -struct local_t {}; - -template -using local_time = std::chrono::time_point; - -using local_seconds = local_time; -using local_days = local_time; - -// types - -struct last_spec { - last_spec() = default; -}; - -class day; -class month; -class year; - -class weekday; -class weekday_indexed; -class weekday_last; - -class month_day; -class month_day_last; -class month_weekday; -class month_weekday_last; - -class year_month; - -class year_month_day; -class year_month_day_last; -class year_month_weekday; -class year_month_weekday_last; - -// date composition operators - -CONSTCD11 year_month operator/(const year& y, const month& m) NOEXCEPT; -CONSTCD11 year_month operator/(const year& y, int m) NOEXCEPT; - -CONSTCD11 month_day operator/(const day& d, const month& m) NOEXCEPT; -CONSTCD11 month_day operator/(const day& d, int m) NOEXCEPT; -CONSTCD11 month_day operator/(const month& m, const day& d) NOEXCEPT; -CONSTCD11 month_day operator/(const month& m, int d) NOEXCEPT; -CONSTCD11 month_day operator/(int m, const day& d) NOEXCEPT; - -CONSTCD11 month_day_last operator/(const month& m, last_spec) NOEXCEPT; -CONSTCD11 month_day_last operator/(int m, last_spec) NOEXCEPT; -CONSTCD11 month_day_last operator/(last_spec, const month& m) NOEXCEPT; -CONSTCD11 month_day_last operator/(last_spec, int m) NOEXCEPT; - -CONSTCD11 month_weekday operator/(const month& m, const weekday_indexed& wdi) NOEXCEPT; -CONSTCD11 month_weekday operator/(int m, const weekday_indexed& wdi) NOEXCEPT; -CONSTCD11 month_weekday operator/(const weekday_indexed& wdi, const month& m) NOEXCEPT; -CONSTCD11 month_weekday operator/(const weekday_indexed& wdi, int m) NOEXCEPT; - -CONSTCD11 month_weekday_last operator/(const month& m, const weekday_last& wdl) NOEXCEPT; -CONSTCD11 month_weekday_last operator/(int m, const weekday_last& wdl) NOEXCEPT; -CONSTCD11 month_weekday_last operator/(const weekday_last& wdl, const month& m) NOEXCEPT; -CONSTCD11 month_weekday_last operator/(const weekday_last& wdl, int m) NOEXCEPT; - -CONSTCD11 year_month_day operator/(const year_month& ym, const day& d) NOEXCEPT; -CONSTCD11 year_month_day operator/(const year_month& ym, int d) NOEXCEPT; -CONSTCD11 year_month_day operator/(const year& y, const month_day& md) NOEXCEPT; -CONSTCD11 year_month_day operator/(int y, const month_day& md) NOEXCEPT; -CONSTCD11 year_month_day operator/(const month_day& md, const year& y) NOEXCEPT; -CONSTCD11 year_month_day operator/(const month_day& md, int y) NOEXCEPT; - -CONSTCD11 -year_month_day_last operator/(const year_month& ym, last_spec) NOEXCEPT; -CONSTCD11 -year_month_day_last operator/(const year& y, const month_day_last& mdl) NOEXCEPT; -CONSTCD11 -year_month_day_last operator/(int y, const month_day_last& mdl) NOEXCEPT; -CONSTCD11 -year_month_day_last operator/(const month_day_last& mdl, const year& y) NOEXCEPT; -CONSTCD11 -year_month_day_last operator/(const month_day_last& mdl, int y) NOEXCEPT; - -CONSTCD11 -year_month_weekday operator/(const year_month& ym, const weekday_indexed& wdi) NOEXCEPT; - -CONSTCD11 -year_month_weekday operator/(const year& y, const month_weekday& mwd) NOEXCEPT; - -CONSTCD11 -year_month_weekday operator/(int y, const month_weekday& mwd) NOEXCEPT; - -CONSTCD11 -year_month_weekday operator/(const month_weekday& mwd, const year& y) NOEXCEPT; - -CONSTCD11 -year_month_weekday operator/(const month_weekday& mwd, int y) NOEXCEPT; - -CONSTCD11 -year_month_weekday_last operator/(const year_month& ym, const weekday_last& wdl) NOEXCEPT; - -CONSTCD11 -year_month_weekday_last operator/(const year& y, const month_weekday_last& mwdl) NOEXCEPT; - -CONSTCD11 -year_month_weekday_last operator/(int y, const month_weekday_last& mwdl) NOEXCEPT; - -CONSTCD11 -year_month_weekday_last operator/(const month_weekday_last& mwdl, const year& y) NOEXCEPT; - -CONSTCD11 -year_month_weekday_last operator/(const month_weekday_last& mwdl, int y) NOEXCEPT; - -// Detailed interface - -// day - -class day { - unsigned char d_; - - public: - day() = default; - explicit CONSTCD11 day(unsigned d) NOEXCEPT; - - CONSTCD14 day& operator++() NOEXCEPT; - CONSTCD14 day operator++(int) NOEXCEPT; - CONSTCD14 day& operator--() NOEXCEPT; - CONSTCD14 day operator--(int) NOEXCEPT; - - CONSTCD14 day& operator+=(const days& d) NOEXCEPT; - CONSTCD14 day& operator-=(const days& d) NOEXCEPT; - - CONSTCD11 explicit operator unsigned() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; -}; - -CONSTCD11 bool operator==(const day& x, const day& y) NOEXCEPT; -CONSTCD11 bool operator!=(const day& x, const day& y) NOEXCEPT; -CONSTCD11 bool operator<(const day& x, const day& y) NOEXCEPT; -CONSTCD11 bool operator>(const day& x, const day& y) NOEXCEPT; -CONSTCD11 bool operator<=(const day& x, const day& y) NOEXCEPT; -CONSTCD11 bool operator>=(const day& x, const day& y) NOEXCEPT; - -CONSTCD11 day operator+(const day& x, const days& y) NOEXCEPT; -CONSTCD11 day operator+(const days& x, const day& y) NOEXCEPT; -CONSTCD11 day operator-(const day& x, const days& y) NOEXCEPT; -CONSTCD11 days operator-(const day& x, const day& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const day& d); - -// month - -class month { - unsigned char m_; - - public: - month() = default; - explicit CONSTCD11 month(unsigned m) NOEXCEPT; - - CONSTCD14 month& operator++() NOEXCEPT; - CONSTCD14 month operator++(int) NOEXCEPT; - CONSTCD14 month& operator--() NOEXCEPT; - CONSTCD14 month operator--(int) NOEXCEPT; - - CONSTCD14 month& operator+=(const months& m) NOEXCEPT; - CONSTCD14 month& operator-=(const months& m) NOEXCEPT; - - CONSTCD11 explicit operator unsigned() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; -}; - -CONSTCD11 bool operator==(const month& x, const month& y) NOEXCEPT; -CONSTCD11 bool operator!=(const month& x, const month& y) NOEXCEPT; -CONSTCD11 bool operator<(const month& x, const month& y) NOEXCEPT; -CONSTCD11 bool operator>(const month& x, const month& y) NOEXCEPT; -CONSTCD11 bool operator<=(const month& x, const month& y) NOEXCEPT; -CONSTCD11 bool operator>=(const month& x, const month& y) NOEXCEPT; - -CONSTCD14 month operator+(const month& x, const months& y) NOEXCEPT; -CONSTCD14 month operator+(const months& x, const month& y) NOEXCEPT; -CONSTCD14 month operator-(const month& x, const months& y) NOEXCEPT; -CONSTCD14 months operator-(const month& x, const month& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const month& m); - -// year - -class year { - int16_t y_; - - public: - year() = default; - explicit CONSTCD11 year(int y) NOEXCEPT; - - CONSTCD14 year& operator++() NOEXCEPT; - CONSTCD14 year operator++(int) NOEXCEPT; - CONSTCD14 year& operator--() NOEXCEPT; - CONSTCD14 year operator--(int) NOEXCEPT; - - CONSTCD14 year& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year& operator-=(const years& y) NOEXCEPT; - - CONSTCD11 year operator-() const NOEXCEPT; - CONSTCD11 year operator+() const NOEXCEPT; - - CONSTCD11 bool is_leap() const NOEXCEPT; - - CONSTCD11 explicit operator int() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; - - static CONSTCD11 year min() NOEXCEPT; - static CONSTCD11 year max() NOEXCEPT; -}; - -CONSTCD11 bool operator==(const year& x, const year& y) NOEXCEPT; -CONSTCD11 bool operator!=(const year& x, const year& y) NOEXCEPT; -CONSTCD11 bool operator<(const year& x, const year& y) NOEXCEPT; -CONSTCD11 bool operator>(const year& x, const year& y) NOEXCEPT; -CONSTCD11 bool operator<=(const year& x, const year& y) NOEXCEPT; -CONSTCD11 bool operator>=(const year& x, const year& y) NOEXCEPT; - -CONSTCD11 year operator+(const year& x, const years& y) NOEXCEPT; -CONSTCD11 year operator+(const years& x, const year& y) NOEXCEPT; -CONSTCD11 year operator-(const year& x, const years& y) NOEXCEPT; -CONSTCD11 years operator-(const year& x, const year& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const year& y); - -// weekday - -class weekday { - unsigned char wd_; - - public: - weekday() = default; - explicit CONSTCD11 weekday(unsigned wd) NOEXCEPT; - CONSTCD11 weekday(const sys_days& dp) NOEXCEPT; - CONSTCD11 explicit weekday(const local_days& dp) NOEXCEPT; - - CONSTCD14 weekday& operator++() NOEXCEPT; - CONSTCD14 weekday operator++(int) NOEXCEPT; - CONSTCD14 weekday& operator--() NOEXCEPT; - CONSTCD14 weekday operator--(int) NOEXCEPT; - - CONSTCD14 weekday& operator+=(const days& d) NOEXCEPT; - CONSTCD14 weekday& operator-=(const days& d) NOEXCEPT; - - CONSTCD11 bool ok() const NOEXCEPT; - - CONSTCD11 weekday_indexed operator[](unsigned index) const NOEXCEPT; - CONSTCD11 weekday_last operator[](last_spec) const NOEXCEPT; - - private: - static CONSTCD11 unsigned char weekday_from_days(int z) NOEXCEPT; - - friend CONSTCD11 bool operator==(const weekday& x, const weekday& y) NOEXCEPT; - friend CONSTCD14 days operator-(const weekday& x, const weekday& y) NOEXCEPT; - friend CONSTCD14 weekday operator+(const weekday& x, const days& y) NOEXCEPT; - template - friend std::basic_ostream& operator<<( - std::basic_ostream& os, const weekday& wd); - friend class weekday_indexed; -}; - -CONSTCD11 bool operator==(const weekday& x, const weekday& y) NOEXCEPT; -CONSTCD11 bool operator!=(const weekday& x, const weekday& y) NOEXCEPT; - -CONSTCD14 weekday operator+(const weekday& x, const days& y) NOEXCEPT; -CONSTCD14 weekday operator+(const days& x, const weekday& y) NOEXCEPT; -CONSTCD14 weekday operator-(const weekday& x, const days& y) NOEXCEPT; -CONSTCD14 days operator-(const weekday& x, const weekday& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const weekday& wd); - -// weekday_indexed - -class weekday_indexed { - unsigned char wd_ : 4; - unsigned char index_ : 4; - - public: - weekday_indexed() = default; - CONSTCD11 weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT; - - CONSTCD11 date::weekday weekday() const NOEXCEPT; - CONSTCD11 unsigned index() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; -}; - -CONSTCD11 bool operator==(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT; -CONSTCD11 bool operator!=(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const weekday_indexed& wdi); - -// weekday_last - -class weekday_last { - date::weekday wd_; - - public: - explicit CONSTCD11 weekday_last(const date::weekday& wd) NOEXCEPT; - - CONSTCD11 date::weekday weekday() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; -}; - -CONSTCD11 bool operator==(const weekday_last& x, const weekday_last& y) NOEXCEPT; -CONSTCD11 bool operator!=(const weekday_last& x, const weekday_last& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const weekday_last& wdl); - -namespace detail { - -struct unspecified_month_disambiguator {}; - -} // namespace detail - -// year_month - -class year_month { - date::year y_; - date::month m_; - - public: - year_month() = default; - CONSTCD11 year_month(const date::year& y, const date::month& m) NOEXCEPT; - - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - - template - CONSTCD14 year_month& operator+=(const months& dm) NOEXCEPT; - template - CONSTCD14 year_month& operator-=(const months& dm) NOEXCEPT; - CONSTCD14 year_month& operator+=(const years& dy) NOEXCEPT; - CONSTCD14 year_month& operator-=(const years& dy) NOEXCEPT; - - CONSTCD11 bool ok() const NOEXCEPT; -}; - -CONSTCD11 bool operator==(const year_month& x, const year_month& y) NOEXCEPT; -CONSTCD11 bool operator!=(const year_month& x, const year_month& y) NOEXCEPT; -CONSTCD11 bool operator<(const year_month& x, const year_month& y) NOEXCEPT; -CONSTCD11 bool operator>(const year_month& x, const year_month& y) NOEXCEPT; -CONSTCD11 bool operator<=(const year_month& x, const year_month& y) NOEXCEPT; -CONSTCD11 bool operator>=(const year_month& x, const year_month& y) NOEXCEPT; - -template -CONSTCD14 year_month operator+(const year_month& ym, const months& dm) NOEXCEPT; -template -CONSTCD14 year_month operator+(const months& dm, const year_month& ym) NOEXCEPT; -template -CONSTCD14 year_month operator-(const year_month& ym, const months& dm) NOEXCEPT; - -CONSTCD11 months operator-(const year_month& x, const year_month& y) NOEXCEPT; -CONSTCD11 year_month operator+(const year_month& ym, const years& dy) NOEXCEPT; -CONSTCD11 year_month operator+(const years& dy, const year_month& ym) NOEXCEPT; -CONSTCD11 year_month operator-(const year_month& ym, const years& dy) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const year_month& ym); - -// month_day - -class month_day { - date::month m_; - date::day d_; - - public: - month_day() = default; - CONSTCD11 month_day(const date::month& m, const date::day& d) NOEXCEPT; - - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::day day() const NOEXCEPT; - - CONSTCD14 bool ok() const NOEXCEPT; -}; - -CONSTCD11 bool operator==(const month_day& x, const month_day& y) NOEXCEPT; -CONSTCD11 bool operator!=(const month_day& x, const month_day& y) NOEXCEPT; -CONSTCD11 bool operator<(const month_day& x, const month_day& y) NOEXCEPT; -CONSTCD11 bool operator>(const month_day& x, const month_day& y) NOEXCEPT; -CONSTCD11 bool operator<=(const month_day& x, const month_day& y) NOEXCEPT; -CONSTCD11 bool operator>=(const month_day& x, const month_day& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const month_day& md); - -// month_day_last - -class month_day_last { - date::month m_; - - public: - CONSTCD11 explicit month_day_last(const date::month& m) NOEXCEPT; - - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; -}; - -CONSTCD11 bool operator==(const month_day_last& x, const month_day_last& y) NOEXCEPT; -CONSTCD11 bool operator!=(const month_day_last& x, const month_day_last& y) NOEXCEPT; -CONSTCD11 bool operator<(const month_day_last& x, const month_day_last& y) NOEXCEPT; -CONSTCD11 bool operator>(const month_day_last& x, const month_day_last& y) NOEXCEPT; -CONSTCD11 bool operator<=(const month_day_last& x, const month_day_last& y) NOEXCEPT; -CONSTCD11 bool operator>=(const month_day_last& x, const month_day_last& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const month_day_last& mdl); - -// month_weekday - -class month_weekday { - date::month m_; - date::weekday_indexed wdi_; - - public: - CONSTCD11 month_weekday(const date::month& m, - const date::weekday_indexed& wdi) NOEXCEPT; - - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; - - CONSTCD11 bool ok() const NOEXCEPT; -}; - -CONSTCD11 bool operator==(const month_weekday& x, const month_weekday& y) NOEXCEPT; -CONSTCD11 bool operator!=(const month_weekday& x, const month_weekday& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const month_weekday& mwd); - -// month_weekday_last - -class month_weekday_last { - date::month m_; - date::weekday_last wdl_; - - public: - CONSTCD11 month_weekday_last(const date::month& m, - const date::weekday_last& wd) NOEXCEPT; - - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; - - CONSTCD11 bool ok() const NOEXCEPT; -}; - -CONSTCD11 -bool operator==(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; -CONSTCD11 -bool operator!=(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const month_weekday_last& mwdl); - -// class year_month_day - -class year_month_day { - date::year y_; - date::month m_; - date::day d_; - - public: - year_month_day() = default; - CONSTCD11 year_month_day(const date::year& y, const date::month& m, - const date::day& d) NOEXCEPT; - CONSTCD14 year_month_day(const year_month_day_last& ymdl) NOEXCEPT; - - CONSTCD14 year_month_day(sys_days dp) NOEXCEPT; - CONSTCD14 explicit year_month_day(local_days dp) NOEXCEPT; - - template - CONSTCD14 year_month_day& operator+=(const months& m) NOEXCEPT; - template - CONSTCD14 year_month_day& operator-=(const months& m) NOEXCEPT; - CONSTCD14 year_month_day& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year_month_day& operator-=(const years& y) NOEXCEPT; - - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::day day() const NOEXCEPT; - - CONSTCD14 operator sys_days() const NOEXCEPT; - CONSTCD14 explicit operator local_days() const NOEXCEPT; - CONSTCD14 bool ok() const NOEXCEPT; - - private: - static CONSTCD14 year_month_day from_days(days dp) NOEXCEPT; - CONSTCD14 days to_days() const NOEXCEPT; -}; - -CONSTCD11 bool operator==(const year_month_day& x, const year_month_day& y) NOEXCEPT; -CONSTCD11 bool operator!=(const year_month_day& x, const year_month_day& y) NOEXCEPT; -CONSTCD11 bool operator<(const year_month_day& x, const year_month_day& y) NOEXCEPT; -CONSTCD11 bool operator>(const year_month_day& x, const year_month_day& y) NOEXCEPT; -CONSTCD11 bool operator<=(const year_month_day& x, const year_month_day& y) NOEXCEPT; -CONSTCD11 bool operator>=(const year_month_day& x, const year_month_day& y) NOEXCEPT; - -template -CONSTCD14 year_month_day operator+(const year_month_day& ymd, const months& dm) NOEXCEPT; -template -CONSTCD14 year_month_day operator+(const months& dm, const year_month_day& ymd) NOEXCEPT; -template -CONSTCD14 year_month_day operator-(const year_month_day& ymd, const months& dm) NOEXCEPT; -CONSTCD11 year_month_day operator+(const year_month_day& ymd, const years& dy) NOEXCEPT; -CONSTCD11 year_month_day operator+(const years& dy, const year_month_day& ymd) NOEXCEPT; -CONSTCD11 year_month_day operator-(const year_month_day& ymd, const years& dy) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const year_month_day& ymd); - -// year_month_day_last - -class year_month_day_last { - date::year y_; - date::month_day_last mdl_; - - public: - CONSTCD11 year_month_day_last(const date::year& y, - const date::month_day_last& mdl) NOEXCEPT; - - template - CONSTCD14 year_month_day_last& operator+=(const months& m) NOEXCEPT; - template - CONSTCD14 year_month_day_last& operator-=(const months& m) NOEXCEPT; - CONSTCD14 year_month_day_last& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year_month_day_last& operator-=(const years& y) NOEXCEPT; - - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::month_day_last month_day_last() const NOEXCEPT; - CONSTCD14 date::day day() const NOEXCEPT; - - CONSTCD14 operator sys_days() const NOEXCEPT; - CONSTCD14 explicit operator local_days() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; -}; - -CONSTCD11 -bool operator==(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; -CONSTCD11 -bool operator!=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; -CONSTCD11 -bool operator<(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; -CONSTCD11 -bool operator>(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; -CONSTCD11 -bool operator<=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; -CONSTCD11 -bool operator>=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; - -template -CONSTCD14 year_month_day_last operator+(const year_month_day_last& ymdl, - const months& dm) NOEXCEPT; - -template -CONSTCD14 year_month_day_last operator+(const months& dm, - const year_month_day_last& ymdl) NOEXCEPT; - -CONSTCD11 -year_month_day_last operator+(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; - -CONSTCD11 -year_month_day_last operator+(const years& dy, const year_month_day_last& ymdl) NOEXCEPT; - -template -CONSTCD14 year_month_day_last operator-(const year_month_day_last& ymdl, - const months& dm) NOEXCEPT; - -CONSTCD11 -year_month_day_last operator-(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const year_month_day_last& ymdl); - -// year_month_weekday - -class year_month_weekday { - date::year y_; - date::month m_; - date::weekday_indexed wdi_; - - public: - year_month_weekday() = default; - CONSTCD11 year_month_weekday(const date::year& y, const date::month& m, - const date::weekday_indexed& wdi) NOEXCEPT; - CONSTCD14 year_month_weekday(const sys_days& dp) NOEXCEPT; - CONSTCD14 explicit year_month_weekday(const local_days& dp) NOEXCEPT; - - template - CONSTCD14 year_month_weekday& operator+=(const months& m) NOEXCEPT; - template - CONSTCD14 year_month_weekday& operator-=(const months& m) NOEXCEPT; - CONSTCD14 year_month_weekday& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year_month_weekday& operator-=(const years& y) NOEXCEPT; - - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::weekday weekday() const NOEXCEPT; - CONSTCD11 unsigned index() const NOEXCEPT; - CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; - - CONSTCD14 operator sys_days() const NOEXCEPT; - CONSTCD14 explicit operator local_days() const NOEXCEPT; - CONSTCD14 bool ok() const NOEXCEPT; - - private: - static CONSTCD14 year_month_weekday from_days(days dp) NOEXCEPT; - CONSTCD14 days to_days() const NOEXCEPT; -}; - -CONSTCD11 -bool operator==(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; -CONSTCD11 -bool operator!=(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; - -template -CONSTCD14 year_month_weekday operator+(const year_month_weekday& ymwd, - const months& dm) NOEXCEPT; - -template -CONSTCD14 year_month_weekday operator+(const months& dm, - const year_month_weekday& ymwd) NOEXCEPT; - -CONSTCD11 -year_month_weekday operator+(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; - -CONSTCD11 -year_month_weekday operator+(const years& dy, const year_month_weekday& ymwd) NOEXCEPT; - -template -CONSTCD14 year_month_weekday operator-(const year_month_weekday& ymwd, - const months& dm) NOEXCEPT; - -CONSTCD11 -year_month_weekday operator-(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const year_month_weekday& ymwdi); - -// year_month_weekday_last - -class year_month_weekday_last { - date::year y_; - date::month m_; - date::weekday_last wdl_; - - public: - CONSTCD11 year_month_weekday_last(const date::year& y, const date::month& m, - const date::weekday_last& wdl) NOEXCEPT; - - template - CONSTCD14 year_month_weekday_last& operator+=(const months& m) NOEXCEPT; - template - CONSTCD14 year_month_weekday_last& operator-=(const months& m) NOEXCEPT; - CONSTCD14 year_month_weekday_last& operator+=(const years& y) NOEXCEPT; - CONSTCD14 year_month_weekday_last& operator-=(const years& y) NOEXCEPT; - - CONSTCD11 date::year year() const NOEXCEPT; - CONSTCD11 date::month month() const NOEXCEPT; - CONSTCD11 date::weekday weekday() const NOEXCEPT; - CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; - - CONSTCD14 operator sys_days() const NOEXCEPT; - CONSTCD14 explicit operator local_days() const NOEXCEPT; - CONSTCD11 bool ok() const NOEXCEPT; - - private: - CONSTCD14 days to_days() const NOEXCEPT; -}; - -CONSTCD11 -bool operator==(const year_month_weekday_last& x, - const year_month_weekday_last& y) NOEXCEPT; - -CONSTCD11 -bool operator!=(const year_month_weekday_last& x, - const year_month_weekday_last& y) NOEXCEPT; - -template -CONSTCD14 year_month_weekday_last operator+(const year_month_weekday_last& ymwdl, - const months& dm) NOEXCEPT; - -template -CONSTCD14 year_month_weekday_last -operator+(const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT; - -CONSTCD11 -year_month_weekday_last operator+(const year_month_weekday_last& ymwdl, - const years& dy) NOEXCEPT; - -CONSTCD11 -year_month_weekday_last operator+(const years& dy, - const year_month_weekday_last& ymwdl) NOEXCEPT; - -template -CONSTCD14 year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, - const months& dm) NOEXCEPT; - -CONSTCD11 -year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, - const years& dy) NOEXCEPT; - -template -std::basic_ostream& operator<<(std::basic_ostream& os, - const year_month_weekday_last& ymwdl); - -#if !defined(_MSC_VER) || (_MSC_VER >= 1900) -inline namespace literals { -CONSTCD11 date::day operator"" _d(unsigned long long d) NOEXCEPT; -CONSTCD11 date::year operator"" _y(unsigned long long y) NOEXCEPT; -} // namespace literals -#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) - -// CONSTDATA date::month January{1}; -// CONSTDATA date::month February{2}; -// CONSTDATA date::month March{3}; -// CONSTDATA date::month April{4}; -// CONSTDATA date::month May{5}; -// CONSTDATA date::month June{6}; -// CONSTDATA date::month July{7}; -// CONSTDATA date::month August{8}; -// CONSTDATA date::month September{9}; -// CONSTDATA date::month October{10}; -// CONSTDATA date::month November{11}; -// CONSTDATA date::month December{12}; -// -// CONSTDATA date::weekday Sunday{0u}; -// CONSTDATA date::weekday Monday{1u}; -// CONSTDATA date::weekday Tuesday{2u}; -// CONSTDATA date::weekday Wednesday{3u}; -// CONSTDATA date::weekday Thursday{4u}; -// CONSTDATA date::weekday Friday{5u}; -// CONSTDATA date::weekday Saturday{6u}; - -#if HAS_VOID_T - -template > -struct is_clock : std::false_type {}; - -template -struct is_clock< - T, std::void_t> - : std::true_type {}; - -#endif // HAS_VOID_T - -//----------------+ -// Implementation | -//----------------+ - -// utilities -namespace detail { - -template > -class save_istream { - protected: - std::basic_ios& is_; - CharT fill_; - std::ios::fmtflags flags_; - std::streamsize width_; - std::basic_ostream* tie_; - std::locale loc_; - - public: - ~save_istream() { - is_.fill(fill_); - is_.flags(flags_); - is_.width(width_); - is_.imbue(loc_); - is_.tie(tie_); - } - - save_istream(const save_istream&) = delete; - save_istream& operator=(const save_istream&) = delete; - - explicit save_istream(std::basic_ios& is) - : is_(is), - fill_(is.fill()), - flags_(is.flags()), - width_(is.width(0)), - tie_(is.tie(nullptr)), - loc_(is.getloc()) { - if (tie_ != nullptr) tie_->flush(); - } -}; - -template > -class save_ostream : private save_istream { - public: - ~save_ostream() { - if ((this->flags_ & std::ios::unitbuf) && -#if __cplusplus >= 201703 - std::uncaught_exceptions() == 0 && -#else - !std::uncaught_exception() && -#endif - this->is_.good()) - this->is_.rdbuf()->pubsync(); - } - - save_ostream(const save_ostream&) = delete; - save_ostream& operator=(const save_ostream&) = delete; - - explicit save_ostream(std::basic_ios& os) - : save_istream(os) {} -}; - -template -struct choose_trunc_type { - static const int digits = std::numeric_limits::digits; - using type = typename std::conditional < digits < 32, std::int32_t, - typename std::conditional::type>::type; -}; - -template -CONSTCD11 inline - typename std::enable_if::value, T>::type - trunc(T t) NOEXCEPT { - return t; -} - -template -CONSTCD14 inline - typename std::enable_if::value, T>::type - trunc(T t) NOEXCEPT { - using std::numeric_limits; - using I = typename choose_trunc_type::type; - CONSTDATA auto digits = numeric_limits::digits; - static_assert(digits < numeric_limits::digits, ""); - CONSTDATA auto max = I{1} << (digits - 1); - CONSTDATA auto min = -max; - const auto negative = t < T{0}; - if (min <= t && t <= max && t != 0 && t == t) { - t = static_cast(static_cast(t)); - if (t == 0 && negative) t = -t; - } - return t; -} - -template -struct static_gcd { - static const std::intmax_t value = static_gcd::value; -}; - -template -struct static_gcd { - static const std::intmax_t value = Xp; -}; - -template <> -struct static_gcd<0, 0> { - static const std::intmax_t value = 1; -}; - -template -struct no_overflow { - private: - static const std::intmax_t gcd_n1_n2 = static_gcd::value; - static const std::intmax_t gcd_d1_d2 = static_gcd::value; - static const std::intmax_t n1 = R1::num / gcd_n1_n2; - static const std::intmax_t d1 = R1::den / gcd_d1_d2; - static const std::intmax_t n2 = R2::num / gcd_n1_n2; - static const std::intmax_t d2 = R2::den / gcd_d1_d2; - static const std::intmax_t max = - -((std::intmax_t(1) << (sizeof(std::intmax_t) * CHAR_BIT - 1)) + 1); - - template - struct mul { // overflow == false - static const std::intmax_t value = Xp * Yp; - }; - - template - struct mul { - static const std::intmax_t value = 1; - }; - - public: - static const bool value = (n1 <= max / d2) && (n2 <= max / d1); - typedef std::ratio::value, mul::value> type; -}; - -} // namespace detail - -// trunc towards zero -template -CONSTCD11 inline - typename std::enable_if::value, - To>::type - trunc(const std::chrono::duration& d) { - return To{detail::trunc(std::chrono::duration_cast(d).count())}; -} - -template -CONSTCD11 inline - typename std::enable_if::value, - To>::type - trunc(const std::chrono::duration& d) { - using std::chrono::duration; - using std::chrono::duration_cast; - using rep = typename std::common_type::type; - return To{detail::trunc(duration_cast(duration_cast>(d)).count())}; -} - -#ifndef HAS_CHRONO_ROUNDING -#if defined(_MSC_FULL_VER) && \ - (_MSC_FULL_VER >= 190023918 || (_MSC_FULL_VER >= 190000000 && defined(__clang__))) -#define HAS_CHRONO_ROUNDING 1 -#elif defined(__cpp_lib_chrono) && __cplusplus > 201402 && __cpp_lib_chrono >= 201510 -#define HAS_CHRONO_ROUNDING 1 -#elif defined(_LIBCPP_VERSION) && __cplusplus > 201402 && _LIBCPP_VERSION >= 3800 -#define HAS_CHRONO_ROUNDING 1 -#else -#define HAS_CHRONO_ROUNDING 0 -#endif -#endif // HAS_CHRONO_ROUNDING - -#if HAS_CHRONO_ROUNDING == 0 - -// round down -template -CONSTCD14 inline - typename std::enable_if::value, - To>::type - floor(const std::chrono::duration& d) { - auto t = trunc(d); - if (t > d) return t - To{1}; - return t; -} - -template -CONSTCD14 inline - typename std::enable_if::value, - To>::type - floor(const std::chrono::duration& d) { - using std::chrono::duration; - using rep = typename std::common_type::type; - return floor(floor>(d)); -} - -// round to nearest, to even on tie -template -CONSTCD14 inline To round(const std::chrono::duration& d) { - auto t0 = floor(d); - auto t1 = t0 + To{1}; - if (t1 == To{0} && t0 < To{0}) t1 = -t1; - auto diff0 = d - t0; - auto diff1 = t1 - d; - if (diff0 == diff1) { - if (t0 - trunc(t0 / 2) * 2 == To{0}) return t0; - return t1; - } - if (diff0 < diff1) return t0; - return t1; -} - -// round up -template -CONSTCD14 inline To ceil(const std::chrono::duration& d) { - auto t = trunc(d); - if (t < d) return t + To{1}; - return t; -} - -template ::is_signed>::type> -CONSTCD11 std::chrono::duration abs(std::chrono::duration d) { - return d >= d.zero() ? d : -d; -} - -// round down -template -CONSTCD11 inline std::chrono::time_point floor( - const std::chrono::time_point& tp) { - using std::chrono::time_point; - return time_point{date::floor(tp.time_since_epoch())}; -} - -// round to nearest, to even on tie -template -CONSTCD11 inline std::chrono::time_point round( - const std::chrono::time_point& tp) { - using std::chrono::time_point; - return time_point{round(tp.time_since_epoch())}; -} - -// round up -template -CONSTCD11 inline std::chrono::time_point ceil( - const std::chrono::time_point& tp) { - using std::chrono::time_point; - return time_point{ceil(tp.time_since_epoch())}; -} - -#else // HAS_CHRONO_ROUNDING == 1 - -using std::chrono::abs; -using std::chrono::ceil; -using std::chrono::floor; -using std::chrono::round; - -#endif // HAS_CHRONO_ROUNDING - -// trunc towards zero -template -CONSTCD11 inline std::chrono::time_point trunc( - const std::chrono::time_point& tp) { - using std::chrono::time_point; - return time_point{trunc(tp.time_since_epoch())}; -} - -// day - -CONSTCD11 inline day::day(unsigned d) NOEXCEPT : d_(static_cast(d)) {} -CONSTCD14 inline day& day::operator++() NOEXCEPT { - ++d_; - return *this; -} -CONSTCD14 inline day day::operator++(int) NOEXCEPT { - auto tmp(*this); - ++(*this); - return tmp; -} -CONSTCD14 inline day& day::operator--() NOEXCEPT { - --d_; - return *this; -} -CONSTCD14 inline day day::operator--(int) NOEXCEPT { - auto tmp(*this); - --(*this); - return tmp; -} -CONSTCD14 inline day& day::operator+=(const days& d) NOEXCEPT { - *this = *this + d; - return *this; -} -CONSTCD14 inline day& day::operator-=(const days& d) NOEXCEPT { - *this = *this - d; - return *this; -} -CONSTCD11 inline day::operator unsigned() const NOEXCEPT { return d_; } -CONSTCD11 inline bool day::ok() const NOEXCEPT { return 1 <= d_ && d_ <= 31; } - -CONSTCD11 -inline bool operator==(const day& x, const day& y) NOEXCEPT { - return static_cast(x) == static_cast(y); -} - -CONSTCD11 -inline bool operator!=(const day& x, const day& y) NOEXCEPT { return !(x == y); } - -CONSTCD11 -inline bool operator<(const day& x, const day& y) NOEXCEPT { - return static_cast(x) < static_cast(y); -} - -CONSTCD11 -inline bool operator>(const day& x, const day& y) NOEXCEPT { return y < x; } - -CONSTCD11 -inline bool operator<=(const day& x, const day& y) NOEXCEPT { return !(y < x); } - -CONSTCD11 -inline bool operator>=(const day& x, const day& y) NOEXCEPT { return !(x < y); } - -CONSTCD11 -inline days operator-(const day& x, const day& y) NOEXCEPT { - return days{ - static_cast(static_cast(x) - static_cast(y))}; -} - -CONSTCD11 -inline day operator+(const day& x, const days& y) NOEXCEPT { - return day{static_cast(x) + static_cast(y.count())}; -} - -CONSTCD11 -inline day operator+(const days& x, const day& y) NOEXCEPT { return y + x; } - -CONSTCD11 -inline day operator-(const day& x, const days& y) NOEXCEPT { return x + -y; } - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const day& d) { - detail::save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << static_cast(d); - if (!d.ok()) os << " is not a valid day"; - return os; -} - -// month - -CONSTCD11 inline month::month(unsigned m) NOEXCEPT : m_(static_cast(m)) {} -CONSTCD14 inline month& month::operator++() NOEXCEPT { - *this += months{1}; - return *this; -} -CONSTCD14 inline month month::operator++(int) NOEXCEPT { - auto tmp(*this); - ++(*this); - return tmp; -} -CONSTCD14 inline month& month::operator--() NOEXCEPT { - *this -= months{1}; - return *this; -} -CONSTCD14 inline month month::operator--(int) NOEXCEPT { - auto tmp(*this); - --(*this); - return tmp; -} - -CONSTCD14 -inline month& month::operator+=(const months& m) NOEXCEPT { - *this = *this + m; - return *this; -} - -CONSTCD14 -inline month& month::operator-=(const months& m) NOEXCEPT { - *this = *this - m; - return *this; -} - -CONSTCD11 inline month::operator unsigned() const NOEXCEPT { return m_; } -CONSTCD11 inline bool month::ok() const NOEXCEPT { return 1 <= m_ && m_ <= 12; } - -CONSTCD11 -inline bool operator==(const month& x, const month& y) NOEXCEPT { - return static_cast(x) == static_cast(y); -} - -CONSTCD11 -inline bool operator!=(const month& x, const month& y) NOEXCEPT { return !(x == y); } - -CONSTCD11 -inline bool operator<(const month& x, const month& y) NOEXCEPT { - return static_cast(x) < static_cast(y); -} - -CONSTCD11 -inline bool operator>(const month& x, const month& y) NOEXCEPT { return y < x; } - -CONSTCD11 -inline bool operator<=(const month& x, const month& y) NOEXCEPT { return !(y < x); } - -CONSTCD11 -inline bool operator>=(const month& x, const month& y) NOEXCEPT { return !(x < y); } - -CONSTCD14 -inline months operator-(const month& x, const month& y) NOEXCEPT { - auto const d = static_cast(x) - static_cast(y); - return months(d <= 11 ? d : d + 12); -} - -CONSTCD14 -inline month operator+(const month& x, const months& y) NOEXCEPT { - auto const mu = static_cast(static_cast(x)) + (y.count() - 1); - auto const yr = (mu >= 0 ? mu : mu - 11) / 12; - return month{static_cast(mu - yr * 12 + 1)}; -} - -CONSTCD14 -inline month operator+(const months& x, const month& y) NOEXCEPT { return y + x; } - -CONSTCD14 -inline month operator-(const month& x, const months& y) NOEXCEPT { return x + -y; } - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const month& m) { - if (m.ok()) { - CharT fmt[] = {'%', 'b', 0}; - os << format(os.getloc(), fmt, m); - } else { - os << static_cast(m) << " is not a valid month"; - } - return os; -} - -// year - -CONSTCD11 inline year::year(int y) NOEXCEPT : y_(static_cast(y)) {} -CONSTCD14 inline year& year::operator++() NOEXCEPT { - ++y_; - return *this; -} -CONSTCD14 inline year year::operator++(int) NOEXCEPT { - auto tmp(*this); - ++(*this); - return tmp; -} -CONSTCD14 inline year& year::operator--() NOEXCEPT { - --y_; - return *this; -} -CONSTCD14 inline year year::operator--(int) NOEXCEPT { - auto tmp(*this); - --(*this); - return tmp; -} -CONSTCD14 inline year& year::operator+=(const years& y) NOEXCEPT { - *this = *this + y; - return *this; -} -CONSTCD14 inline year& year::operator-=(const years& y) NOEXCEPT { - *this = *this - y; - return *this; -} -CONSTCD11 inline year year::operator-() const NOEXCEPT { return year{-y_}; } -CONSTCD11 inline year year::operator+() const NOEXCEPT { return *this; } - -CONSTCD11 -inline bool year::is_leap() const NOEXCEPT { - return y_ % 4 == 0 && (y_ % 100 != 0 || y_ % 400 == 0); -} - -CONSTCD11 inline year::operator int() const NOEXCEPT { return y_; } - -CONSTCD11 -inline bool year::ok() const NOEXCEPT { - return y_ != std::numeric_limits::min(); -} - -CONSTCD11 -inline year year::min() NOEXCEPT { return year{-32767}; } - -CONSTCD11 -inline year year::max() NOEXCEPT { return year{32767}; } - -CONSTCD11 -inline bool operator==(const year& x, const year& y) NOEXCEPT { - return static_cast(x) == static_cast(y); -} - -CONSTCD11 -inline bool operator!=(const year& x, const year& y) NOEXCEPT { return !(x == y); } - -CONSTCD11 -inline bool operator<(const year& x, const year& y) NOEXCEPT { - return static_cast(x) < static_cast(y); -} - -CONSTCD11 -inline bool operator>(const year& x, const year& y) NOEXCEPT { return y < x; } - -CONSTCD11 -inline bool operator<=(const year& x, const year& y) NOEXCEPT { return !(y < x); } - -CONSTCD11 -inline bool operator>=(const year& x, const year& y) NOEXCEPT { return !(x < y); } - -CONSTCD11 -inline years operator-(const year& x, const year& y) NOEXCEPT { - return years{static_cast(x) - static_cast(y)}; -} - -CONSTCD11 -inline year operator+(const year& x, const years& y) NOEXCEPT { - return year{static_cast(x) + y.count()}; -} - -CONSTCD11 -inline year operator+(const years& x, const year& y) NOEXCEPT { return y + x; } - -CONSTCD11 -inline year operator-(const year& x, const years& y) NOEXCEPT { - return year{static_cast(x) - y.count()}; -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const year& y) { - detail::save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::internal); - os.width(4 + (y < year{0})); - os << static_cast(y); - if (!y.ok()) os << " is not a valid year"; - return os; -} - -// weekday - -CONSTCD11 -inline unsigned char weekday::weekday_from_days(int z) NOEXCEPT { - return static_cast( - static_cast(z >= -4 ? (z + 4) % 7 : (z + 5) % 7 + 6)); -} - -CONSTCD11 -inline weekday::weekday(unsigned wd) NOEXCEPT - : wd_(static_cast(wd != 7 ? wd : 0)) {} - -CONSTCD11 -inline weekday::weekday(const sys_days& dp) NOEXCEPT - : wd_(weekday_from_days(dp.time_since_epoch().count())) {} - -CONSTCD11 -inline weekday::weekday(const local_days& dp) NOEXCEPT - : wd_(weekday_from_days(dp.time_since_epoch().count())) {} - -CONSTCD14 inline weekday& weekday::operator++() NOEXCEPT { - *this += days{1}; - return *this; -} -CONSTCD14 inline weekday weekday::operator++(int) NOEXCEPT { - auto tmp(*this); - ++(*this); - return tmp; -} -CONSTCD14 inline weekday& weekday::operator--() NOEXCEPT { - *this -= days{1}; - return *this; -} -CONSTCD14 inline weekday weekday::operator--(int) NOEXCEPT { - auto tmp(*this); - --(*this); - return tmp; -} - -CONSTCD14 -inline weekday& weekday::operator+=(const days& d) NOEXCEPT { - *this = *this + d; - return *this; -} - -CONSTCD14 -inline weekday& weekday::operator-=(const days& d) NOEXCEPT { - *this = *this - d; - return *this; -} - -CONSTCD11 inline bool weekday::ok() const NOEXCEPT { return wd_ <= 6; } - -CONSTCD11 -inline bool operator==(const weekday& x, const weekday& y) NOEXCEPT { - return x.wd_ == y.wd_; -} - -CONSTCD11 -inline bool operator!=(const weekday& x, const weekday& y) NOEXCEPT { return !(x == y); } - -CONSTCD14 -inline days operator-(const weekday& x, const weekday& y) NOEXCEPT { - auto const wdu = x.wd_ - y.wd_; - auto const wk = (wdu >= 0 ? wdu : wdu - 6) / 7; - return days{wdu - wk * 7}; -} - -CONSTCD14 -inline weekday operator+(const weekday& x, const days& y) NOEXCEPT { - auto const wdu = static_cast(static_cast(x.wd_)) + y.count(); - auto const wk = (wdu >= 0 ? wdu : wdu - 6) / 7; - return weekday{static_cast(wdu - wk * 7)}; -} - -CONSTCD14 -inline weekday operator+(const days& x, const weekday& y) NOEXCEPT { return y + x; } - -CONSTCD14 -inline weekday operator-(const weekday& x, const days& y) NOEXCEPT { return x + -y; } - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const weekday& wd) { - if (wd.ok()) { - CharT fmt[] = {'%', 'a', 0}; - os << format(fmt, wd); - } else { - os << static_cast(wd.wd_) << " is not a valid weekday"; - } - return os; -} - -#if !defined(_MSC_VER) || (_MSC_VER >= 1900) -inline namespace literals { -CONSTCD11 -inline date::day operator"" _d(unsigned long long d) NOEXCEPT { - return date::day{static_cast(d)}; -} - -CONSTCD11 -inline date::year operator"" _y(unsigned long long y) NOEXCEPT { - return date::year(static_cast(y)); -} -#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) - -CONSTDATA date::last_spec last{}; - -CONSTDATA date::month jan{1}; -CONSTDATA date::month feb{2}; -CONSTDATA date::month mar{3}; -CONSTDATA date::month apr{4}; -CONSTDATA date::month may{5}; -CONSTDATA date::month jun{6}; -CONSTDATA date::month jul{7}; -CONSTDATA date::month aug{8}; -CONSTDATA date::month sep{9}; -CONSTDATA date::month oct{10}; -CONSTDATA date::month nov{11}; -CONSTDATA date::month dec{12}; - -CONSTDATA date::weekday sun{0u}; -CONSTDATA date::weekday mon{1u}; -CONSTDATA date::weekday tue{2u}; -CONSTDATA date::weekday wed{3u}; -CONSTDATA date::weekday thu{4u}; -CONSTDATA date::weekday fri{5u}; -CONSTDATA date::weekday sat{6u}; - -#if !defined(_MSC_VER) || (_MSC_VER >= 1900) -} // inline namespace literals -#endif - -CONSTDATA date::month January{1}; -CONSTDATA date::month February{2}; -CONSTDATA date::month March{3}; -CONSTDATA date::month April{4}; -CONSTDATA date::month May{5}; -CONSTDATA date::month June{6}; -CONSTDATA date::month July{7}; -CONSTDATA date::month August{8}; -CONSTDATA date::month September{9}; -CONSTDATA date::month October{10}; -CONSTDATA date::month November{11}; -CONSTDATA date::month December{12}; - -CONSTDATA date::weekday Monday{1}; -CONSTDATA date::weekday Tuesday{2}; -CONSTDATA date::weekday Wednesday{3}; -CONSTDATA date::weekday Thursday{4}; -CONSTDATA date::weekday Friday{5}; -CONSTDATA date::weekday Saturday{6}; -CONSTDATA date::weekday Sunday{7}; - -// weekday_indexed - -CONSTCD11 -inline weekday weekday_indexed::weekday() const NOEXCEPT { - return date::weekday{static_cast(wd_)}; -} - -CONSTCD11 inline unsigned weekday_indexed::index() const NOEXCEPT { return index_; } - -CONSTCD11 -inline bool weekday_indexed::ok() const NOEXCEPT { - return weekday().ok() && 1 <= index_ && index_ <= 5; -} - -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wconversion" -#endif // __GNUC__ - -CONSTCD11 -inline weekday_indexed::weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT - : wd_(static_cast(static_cast(wd.wd_))), - index_(static_cast(index)) {} - -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif // __GNUC__ - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const weekday_indexed& wdi) { - os << wdi.weekday() << '[' << wdi.index(); - if (!(1 <= wdi.index() && wdi.index() <= 5)) os << " is not a valid index"; - os << ']'; - return os; -} - -CONSTCD11 -inline weekday_indexed weekday::operator[](unsigned index) const NOEXCEPT { - return {*this, index}; -} - -CONSTCD11 -inline bool operator==(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT { - return x.weekday() == y.weekday() && x.index() == y.index(); -} - -CONSTCD11 -inline bool operator!=(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT { - return !(x == y); -} - -// weekday_last - -CONSTCD11 inline date::weekday weekday_last::weekday() const NOEXCEPT { return wd_; } -CONSTCD11 inline bool weekday_last::ok() const NOEXCEPT { return wd_.ok(); } -CONSTCD11 inline weekday_last::weekday_last(const date::weekday& wd) NOEXCEPT : wd_(wd) {} - -CONSTCD11 -inline bool operator==(const weekday_last& x, const weekday_last& y) NOEXCEPT { - return x.weekday() == y.weekday(); -} - -CONSTCD11 -inline bool operator!=(const weekday_last& x, const weekday_last& y) NOEXCEPT { - return !(x == y); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const weekday_last& wdl) { - return os << wdl.weekday() << "[last]"; -} - -CONSTCD11 -inline weekday_last weekday::operator[](last_spec) const NOEXCEPT { - return weekday_last{*this}; -} - -// year_month - -CONSTCD11 -inline year_month::year_month(const date::year& y, const date::month& m) NOEXCEPT - : y_(y), - m_(m) {} - -CONSTCD11 inline year year_month::year() const NOEXCEPT { return y_; } -CONSTCD11 inline month year_month::month() const NOEXCEPT { return m_; } -CONSTCD11 inline bool year_month::ok() const NOEXCEPT { return y_.ok() && m_.ok(); } - -template -CONSTCD14 inline year_month& year_month::operator+=(const months& dm) NOEXCEPT { - *this = *this + dm; - return *this; -} - -template -CONSTCD14 inline year_month& year_month::operator-=(const months& dm) NOEXCEPT { - *this = *this - dm; - return *this; -} - -CONSTCD14 -inline year_month& year_month::operator+=(const years& dy) NOEXCEPT { - *this = *this + dy; - return *this; -} - -CONSTCD14 -inline year_month& year_month::operator-=(const years& dy) NOEXCEPT { - *this = *this - dy; - return *this; -} - -CONSTCD11 -inline bool operator==(const year_month& x, const year_month& y) NOEXCEPT { - return x.year() == y.year() && x.month() == y.month(); -} - -CONSTCD11 -inline bool operator!=(const year_month& x, const year_month& y) NOEXCEPT { - return !(x == y); -} - -CONSTCD11 -inline bool operator<(const year_month& x, const year_month& y) NOEXCEPT { - return x.year() < y.year() ? true - : (x.year() > y.year() ? false : (x.month() < y.month())); -} - -CONSTCD11 -inline bool operator>(const year_month& x, const year_month& y) NOEXCEPT { return y < x; } - -CONSTCD11 -inline bool operator<=(const year_month& x, const year_month& y) NOEXCEPT { - return !(y < x); -} - -CONSTCD11 -inline bool operator>=(const year_month& x, const year_month& y) NOEXCEPT { - return !(x < y); -} - -template -CONSTCD14 inline year_month operator+(const year_month& ym, const months& dm) NOEXCEPT { - auto dmi = static_cast(static_cast(ym.month())) - 1 + dm.count(); - auto dy = (dmi >= 0 ? dmi : dmi - 11) / 12; - dmi = dmi - dy * 12 + 1; - return (ym.year() + years(dy)) / month(static_cast(dmi)); -} - -template -CONSTCD14 inline year_month operator+(const months& dm, const year_month& ym) NOEXCEPT { - return ym + dm; -} - -template -CONSTCD14 inline year_month operator-(const year_month& ym, const months& dm) NOEXCEPT { - return ym + -dm; -} - -CONSTCD11 -inline months operator-(const year_month& x, const year_month& y) NOEXCEPT { - return (x.year() - y.year()) + - months(static_cast(x.month()) - static_cast(y.month())); -} - -CONSTCD11 -inline year_month operator+(const year_month& ym, const years& dy) NOEXCEPT { - return (ym.year() + dy) / ym.month(); -} - -CONSTCD11 -inline year_month operator+(const years& dy, const year_month& ym) NOEXCEPT { - return ym + dy; -} - -CONSTCD11 -inline year_month operator-(const year_month& ym, const years& dy) NOEXCEPT { - return ym + -dy; -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const year_month& ym) { - return os << ym.year() << '/' << ym.month(); -} - -// month_day - -CONSTCD11 -inline month_day::month_day(const date::month& m, const date::day& d) NOEXCEPT : m_(m), - d_(d) {} - -CONSTCD11 inline date::month month_day::month() const NOEXCEPT { return m_; } -CONSTCD11 inline date::day month_day::day() const NOEXCEPT { return d_; } - -CONSTCD14 -inline bool month_day::ok() const NOEXCEPT { - CONSTDATA date::day d[] = {date::day(31), date::day(29), date::day(31), date::day(30), - date::day(31), date::day(30), date::day(31), date::day(31), - date::day(30), date::day(31), date::day(30), date::day(31)}; - return m_.ok() && date::day{1} <= d_ && d_ <= d[static_cast(m_) - 1]; -} - -CONSTCD11 -inline bool operator==(const month_day& x, const month_day& y) NOEXCEPT { - return x.month() == y.month() && x.day() == y.day(); -} - -CONSTCD11 -inline bool operator!=(const month_day& x, const month_day& y) NOEXCEPT { - return !(x == y); -} - -CONSTCD11 -inline bool operator<(const month_day& x, const month_day& y) NOEXCEPT { - return x.month() < y.month() ? true - : (x.month() > y.month() ? false : (x.day() < y.day())); -} - -CONSTCD11 -inline bool operator>(const month_day& x, const month_day& y) NOEXCEPT { return y < x; } - -CONSTCD11 -inline bool operator<=(const month_day& x, const month_day& y) NOEXCEPT { - return !(y < x); -} - -CONSTCD11 -inline bool operator>=(const month_day& x, const month_day& y) NOEXCEPT { - return !(x < y); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const month_day& md) { - return os << md.month() << '/' << md.day(); -} - -// month_day_last - -CONSTCD11 inline month month_day_last::month() const NOEXCEPT { return m_; } -CONSTCD11 inline bool month_day_last::ok() const NOEXCEPT { return m_.ok(); } -CONSTCD11 inline month_day_last::month_day_last(const date::month& m) NOEXCEPT : m_(m) {} - -CONSTCD11 -inline bool operator==(const month_day_last& x, const month_day_last& y) NOEXCEPT { - return x.month() == y.month(); -} - -CONSTCD11 -inline bool operator!=(const month_day_last& x, const month_day_last& y) NOEXCEPT { - return !(x == y); -} - -CONSTCD11 -inline bool operator<(const month_day_last& x, const month_day_last& y) NOEXCEPT { - return x.month() < y.month(); -} - -CONSTCD11 -inline bool operator>(const month_day_last& x, const month_day_last& y) NOEXCEPT { - return y < x; -} - -CONSTCD11 -inline bool operator<=(const month_day_last& x, const month_day_last& y) NOEXCEPT { - return !(y < x); -} - -CONSTCD11 -inline bool operator>=(const month_day_last& x, const month_day_last& y) NOEXCEPT { - return !(x < y); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const month_day_last& mdl) { - return os << mdl.month() << "/last"; -} - -// month_weekday - -CONSTCD11 -inline month_weekday::month_weekday(const date::month& m, - const date::weekday_indexed& wdi) NOEXCEPT - : m_(m), - wdi_(wdi) {} - -CONSTCD11 inline month month_weekday::month() const NOEXCEPT { return m_; } - -CONSTCD11 -inline weekday_indexed month_weekday::weekday_indexed() const NOEXCEPT { return wdi_; } - -CONSTCD11 -inline bool month_weekday::ok() const NOEXCEPT { return m_.ok() && wdi_.ok(); } - -CONSTCD11 -inline bool operator==(const month_weekday& x, const month_weekday& y) NOEXCEPT { - return x.month() == y.month() && x.weekday_indexed() == y.weekday_indexed(); -} - -CONSTCD11 -inline bool operator!=(const month_weekday& x, const month_weekday& y) NOEXCEPT { - return !(x == y); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const month_weekday& mwd) { - return os << mwd.month() << '/' << mwd.weekday_indexed(); -} - -// month_weekday_last - -CONSTCD11 -inline month_weekday_last::month_weekday_last(const date::month& m, - const date::weekday_last& wdl) NOEXCEPT - : m_(m), - wdl_(wdl) {} - -CONSTCD11 inline month month_weekday_last::month() const NOEXCEPT { return m_; } - -CONSTCD11 -inline weekday_last month_weekday_last::weekday_last() const NOEXCEPT { return wdl_; } - -CONSTCD11 -inline bool month_weekday_last::ok() const NOEXCEPT { return m_.ok() && wdl_.ok(); } - -CONSTCD11 -inline bool operator==(const month_weekday_last& x, - const month_weekday_last& y) NOEXCEPT { - return x.month() == y.month() && x.weekday_last() == y.weekday_last(); -} - -CONSTCD11 -inline bool operator!=(const month_weekday_last& x, - const month_weekday_last& y) NOEXCEPT { - return !(x == y); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const month_weekday_last& mwdl) { - return os << mwdl.month() << '/' << mwdl.weekday_last(); -} - -// year_month_day_last - -CONSTCD11 -inline year_month_day_last::year_month_day_last(const date::year& y, - const date::month_day_last& mdl) NOEXCEPT - : y_(y), - mdl_(mdl) {} - -template -CONSTCD14 inline year_month_day_last& year_month_day_last::operator+=( - const months& m) NOEXCEPT { - *this = *this + m; - return *this; -} - -template -CONSTCD14 inline year_month_day_last& year_month_day_last::operator-=( - const months& m) NOEXCEPT { - *this = *this - m; - return *this; -} - -CONSTCD14 -inline year_month_day_last& year_month_day_last::operator+=(const years& y) NOEXCEPT { - *this = *this + y; - return *this; -} - -CONSTCD14 -inline year_month_day_last& year_month_day_last::operator-=(const years& y) NOEXCEPT { - *this = *this - y; - return *this; -} - -CONSTCD11 inline year year_month_day_last::year() const NOEXCEPT { return y_; } -CONSTCD11 inline month year_month_day_last::month() const NOEXCEPT { - return mdl_.month(); -} - -CONSTCD11 -inline month_day_last year_month_day_last::month_day_last() const NOEXCEPT { - return mdl_; -} - -CONSTCD14 -inline day year_month_day_last::day() const NOEXCEPT { - CONSTDATA date::day d[] = {date::day(31), date::day(28), date::day(31), date::day(30), - date::day(31), date::day(30), date::day(31), date::day(31), - date::day(30), date::day(31), date::day(30), date::day(31)}; - return month() != February || !y_.is_leap() ? d[static_cast(month()) - 1] - : date::day{29}; -} - -CONSTCD14 -inline year_month_day_last::operator sys_days() const NOEXCEPT { - return sys_days(year() / month() / day()); -} - -CONSTCD14 -inline year_month_day_last::operator local_days() const NOEXCEPT { - return local_days(year() / month() / day()); -} - -CONSTCD11 -inline bool year_month_day_last::ok() const NOEXCEPT { return y_.ok() && mdl_.ok(); } - -CONSTCD11 -inline bool operator==(const year_month_day_last& x, - const year_month_day_last& y) NOEXCEPT { - return x.year() == y.year() && x.month_day_last() == y.month_day_last(); -} - -CONSTCD11 -inline bool operator!=(const year_month_day_last& x, - const year_month_day_last& y) NOEXCEPT { - return !(x == y); -} - -CONSTCD11 -inline bool operator<(const year_month_day_last& x, - const year_month_day_last& y) NOEXCEPT { - return x.year() < y.year() - ? true - : (x.year() > y.year() ? false : (x.month_day_last() < y.month_day_last())); -} - -CONSTCD11 -inline bool operator>(const year_month_day_last& x, - const year_month_day_last& y) NOEXCEPT { - return y < x; -} - -CONSTCD11 -inline bool operator<=(const year_month_day_last& x, - const year_month_day_last& y) NOEXCEPT { - return !(y < x); -} - -CONSTCD11 -inline bool operator>=(const year_month_day_last& x, - const year_month_day_last& y) NOEXCEPT { - return !(x < y); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const year_month_day_last& ymdl) { - return os << ymdl.year() << '/' << ymdl.month_day_last(); -} - -template -CONSTCD14 inline year_month_day_last operator+(const year_month_day_last& ymdl, - const months& dm) NOEXCEPT { - return (ymdl.year() / ymdl.month() + dm) / last; -} - -template -CONSTCD14 inline year_month_day_last operator+(const months& dm, - const year_month_day_last& ymdl) NOEXCEPT { - return ymdl + dm; -} - -template -CONSTCD14 inline year_month_day_last operator-(const year_month_day_last& ymdl, - const months& dm) NOEXCEPT { - return ymdl + (-dm); -} - -CONSTCD11 -inline year_month_day_last operator+(const year_month_day_last& ymdl, - const years& dy) NOEXCEPT { - return {ymdl.year() + dy, ymdl.month_day_last()}; -} - -CONSTCD11 -inline year_month_day_last operator+(const years& dy, - const year_month_day_last& ymdl) NOEXCEPT { - return ymdl + dy; -} - -CONSTCD11 -inline year_month_day_last operator-(const year_month_day_last& ymdl, - const years& dy) NOEXCEPT { - return ymdl + (-dy); -} - -// year_month_day - -CONSTCD11 -inline year_month_day::year_month_day(const date::year& y, const date::month& m, - const date::day& d) NOEXCEPT : y_(y), - m_(m), - d_(d) {} - -CONSTCD14 -inline year_month_day::year_month_day(const year_month_day_last& ymdl) NOEXCEPT - : y_(ymdl.year()), - m_(ymdl.month()), - d_(ymdl.day()) {} - -CONSTCD14 -inline year_month_day::year_month_day(sys_days dp) NOEXCEPT - : year_month_day(from_days(dp.time_since_epoch())) {} - -CONSTCD14 -inline year_month_day::year_month_day(local_days dp) NOEXCEPT - : year_month_day(from_days(dp.time_since_epoch())) {} - -CONSTCD11 inline year year_month_day::year() const NOEXCEPT { return y_; } -CONSTCD11 inline month year_month_day::month() const NOEXCEPT { return m_; } -CONSTCD11 inline day year_month_day::day() const NOEXCEPT { return d_; } - -template -CONSTCD14 inline year_month_day& year_month_day::operator+=(const months& m) NOEXCEPT { - *this = *this + m; - return *this; -} - -template -CONSTCD14 inline year_month_day& year_month_day::operator-=(const months& m) NOEXCEPT { - *this = *this - m; - return *this; -} - -CONSTCD14 -inline year_month_day& year_month_day::operator+=(const years& y) NOEXCEPT { - *this = *this + y; - return *this; -} - -CONSTCD14 -inline year_month_day& year_month_day::operator-=(const years& y) NOEXCEPT { - *this = *this - y; - return *this; -} - -CONSTCD14 -inline days year_month_day::to_days() const NOEXCEPT { - static_assert(std::numeric_limits::digits >= 18, - "This algorithm has not been ported to a 16 bit unsigned integer"); - static_assert(std::numeric_limits::digits >= 20, - "This algorithm has not been ported to a 16 bit signed integer"); - auto const y = static_cast(y_) - (m_ <= February); - auto const m = static_cast(m_); - auto const d = static_cast(d_); - auto const era = (y >= 0 ? y : y - 399) / 400; - auto const yoe = static_cast(y - era * 400); // [0, 399] - auto const doy = (153 * (m > 2 ? m - 3 : m + 9) + 2) / 5 + d - 1; // [0, 365] - auto const doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096] - return days{era * 146097 + static_cast(doe) - 719468}; -} - -CONSTCD14 -inline year_month_day::operator sys_days() const NOEXCEPT { return sys_days{to_days()}; } - -CONSTCD14 -inline year_month_day::operator local_days() const NOEXCEPT { - return local_days{to_days()}; -} - -CONSTCD14 -inline bool year_month_day::ok() const NOEXCEPT { - if (!(y_.ok() && m_.ok())) return false; - return date::day{1} <= d_ && d_ <= (y_ / m_ / last).day(); -} - -CONSTCD11 -inline bool operator==(const year_month_day& x, const year_month_day& y) NOEXCEPT { - return x.year() == y.year() && x.month() == y.month() && x.day() == y.day(); -} - -CONSTCD11 -inline bool operator!=(const year_month_day& x, const year_month_day& y) NOEXCEPT { - return !(x == y); -} - -CONSTCD11 -inline bool operator<(const year_month_day& x, const year_month_day& y) NOEXCEPT { - return x.year() < y.year() - ? true - : (x.year() > y.year() - ? false - : (x.month() < y.month() - ? true - : (x.month() > y.month() ? false : (x.day() < y.day())))); -} - -CONSTCD11 -inline bool operator>(const year_month_day& x, const year_month_day& y) NOEXCEPT { - return y < x; -} - -CONSTCD11 -inline bool operator<=(const year_month_day& x, const year_month_day& y) NOEXCEPT { - return !(y < x); -} - -CONSTCD11 -inline bool operator>=(const year_month_day& x, const year_month_day& y) NOEXCEPT { - return !(x < y); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const year_month_day& ymd) { - detail::save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os << ymd.year() << '-'; - os.width(2); - os << static_cast(ymd.month()) << '-'; - os << ymd.day(); - if (!ymd.ok()) os << " is not a valid date"; - return os; -} - -CONSTCD14 -inline year_month_day year_month_day::from_days(days dp) NOEXCEPT { - static_assert(std::numeric_limits::digits >= 18, - "This algorithm has not been ported to a 16 bit unsigned integer"); - static_assert(std::numeric_limits::digits >= 20, - "This algorithm has not been ported to a 16 bit signed integer"); - auto const z = dp.count() + 719468; - auto const era = (z >= 0 ? z : z - 146096) / 146097; - auto const doe = static_cast(z - era * 146097); // [0, 146096] - auto const yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // [0, 399] - auto const y = static_cast(yoe) + era * 400; - auto const doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] - auto const mp = (5 * doy + 2) / 153; // [0, 11] - auto const d = doy - (153 * mp + 2) / 5 + 1; // [1, 31] - auto const m = mp < 10 ? mp + 3 : mp - 9; // [1, 12] - return year_month_day{date::year{y + (m <= 2)}, date::month(m), date::day(d)}; -} - -template -CONSTCD14 inline year_month_day operator+(const year_month_day& ymd, - const months& dm) NOEXCEPT { - return (ymd.year() / ymd.month() + dm) / ymd.day(); -} - -template -CONSTCD14 inline year_month_day operator+(const months& dm, - const year_month_day& ymd) NOEXCEPT { - return ymd + dm; -} - -template -CONSTCD14 inline year_month_day operator-(const year_month_day& ymd, - const months& dm) NOEXCEPT { - return ymd + (-dm); -} - -CONSTCD11 -inline year_month_day operator+(const year_month_day& ymd, const years& dy) NOEXCEPT { - return (ymd.year() + dy) / ymd.month() / ymd.day(); -} - -CONSTCD11 -inline year_month_day operator+(const years& dy, const year_month_day& ymd) NOEXCEPT { - return ymd + dy; -} - -CONSTCD11 -inline year_month_day operator-(const year_month_day& ymd, const years& dy) NOEXCEPT { - return ymd + (-dy); -} - -// year_month_weekday - -CONSTCD11 -inline year_month_weekday::year_month_weekday(const date::year& y, const date::month& m, - const date::weekday_indexed& wdi) NOEXCEPT - : y_(y), - m_(m), - wdi_(wdi) {} - -CONSTCD14 -inline year_month_weekday::year_month_weekday(const sys_days& dp) NOEXCEPT - : year_month_weekday(from_days(dp.time_since_epoch())) {} - -CONSTCD14 -inline year_month_weekday::year_month_weekday(const local_days& dp) NOEXCEPT - : year_month_weekday(from_days(dp.time_since_epoch())) {} - -template -CONSTCD14 inline year_month_weekday& year_month_weekday::operator+=( - const months& m) NOEXCEPT { - *this = *this + m; - return *this; -} - -template -CONSTCD14 inline year_month_weekday& year_month_weekday::operator-=( - const months& m) NOEXCEPT { - *this = *this - m; - return *this; -} - -CONSTCD14 -inline year_month_weekday& year_month_weekday::operator+=(const years& y) NOEXCEPT { - *this = *this + y; - return *this; -} - -CONSTCD14 -inline year_month_weekday& year_month_weekday::operator-=(const years& y) NOEXCEPT { - *this = *this - y; - return *this; -} - -CONSTCD11 inline year year_month_weekday::year() const NOEXCEPT { return y_; } -CONSTCD11 inline month year_month_weekday::month() const NOEXCEPT { return m_; } - -CONSTCD11 -inline weekday year_month_weekday::weekday() const NOEXCEPT { return wdi_.weekday(); } - -CONSTCD11 -inline unsigned year_month_weekday::index() const NOEXCEPT { return wdi_.index(); } - -CONSTCD11 -inline weekday_indexed year_month_weekday::weekday_indexed() const NOEXCEPT { - return wdi_; -} - -CONSTCD14 -inline year_month_weekday::operator sys_days() const NOEXCEPT { - return sys_days{to_days()}; -} - -CONSTCD14 -inline year_month_weekday::operator local_days() const NOEXCEPT { - return local_days{to_days()}; -} - -CONSTCD14 -inline bool year_month_weekday::ok() const NOEXCEPT { - if (!y_.ok() || !m_.ok() || !wdi_.weekday().ok() || wdi_.index() < 1) return false; - if (wdi_.index() <= 4) return true; - auto d2 = wdi_.weekday() - date::weekday(static_cast(y_ / m_ / 1)) + - days((wdi_.index() - 1) * 7 + 1); - return static_cast(d2.count()) <= - static_cast((y_ / m_ / last).day()); -} - -CONSTCD14 -inline year_month_weekday year_month_weekday::from_days(days d) NOEXCEPT { - sys_days dp{d}; - auto const wd = date::weekday(dp); - auto const ymd = year_month_day(dp); - return {ymd.year(), ymd.month(), wd[(static_cast(ymd.day()) - 1) / 7 + 1]}; -} - -CONSTCD14 -inline days year_month_weekday::to_days() const NOEXCEPT { - auto d = sys_days(y_ / m_ / 1); - return (d + (wdi_.weekday() - date::weekday(d) + days{(wdi_.index() - 1) * 7})) - .time_since_epoch(); -} - -CONSTCD11 -inline bool operator==(const year_month_weekday& x, - const year_month_weekday& y) NOEXCEPT { - return x.year() == y.year() && x.month() == y.month() && - x.weekday_indexed() == y.weekday_indexed(); -} - -CONSTCD11 -inline bool operator!=(const year_month_weekday& x, - const year_month_weekday& y) NOEXCEPT { - return !(x == y); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const year_month_weekday& ymwdi) { - return os << ymwdi.year() << '/' << ymwdi.month() << '/' << ymwdi.weekday_indexed(); -} - -template -CONSTCD14 inline year_month_weekday operator+(const year_month_weekday& ymwd, - const months& dm) NOEXCEPT { - return (ymwd.year() / ymwd.month() + dm) / ymwd.weekday_indexed(); -} - -template -CONSTCD14 inline year_month_weekday operator+(const months& dm, - const year_month_weekday& ymwd) NOEXCEPT { - return ymwd + dm; -} - -template -CONSTCD14 inline year_month_weekday operator-(const year_month_weekday& ymwd, - const months& dm) NOEXCEPT { - return ymwd + (-dm); -} - -CONSTCD11 -inline year_month_weekday operator+(const year_month_weekday& ymwd, - const years& dy) NOEXCEPT { - return {ymwd.year() + dy, ymwd.month(), ymwd.weekday_indexed()}; -} - -CONSTCD11 -inline year_month_weekday operator+(const years& dy, - const year_month_weekday& ymwd) NOEXCEPT { - return ymwd + dy; -} - -CONSTCD11 -inline year_month_weekday operator-(const year_month_weekday& ymwd, - const years& dy) NOEXCEPT { - return ymwd + (-dy); -} - -// year_month_weekday_last - -CONSTCD11 -inline year_month_weekday_last::year_month_weekday_last( - const date::year& y, const date::month& m, const date::weekday_last& wdl) NOEXCEPT - : y_(y), - m_(m), - wdl_(wdl) {} - -template -CONSTCD14 inline year_month_weekday_last& year_month_weekday_last::operator+=( - const months& m) NOEXCEPT { - *this = *this + m; - return *this; -} - -template -CONSTCD14 inline year_month_weekday_last& year_month_weekday_last::operator-=( - const months& m) NOEXCEPT { - *this = *this - m; - return *this; -} - -CONSTCD14 -inline year_month_weekday_last& year_month_weekday_last::operator+=( - const years& y) NOEXCEPT { - *this = *this + y; - return *this; -} - -CONSTCD14 -inline year_month_weekday_last& year_month_weekday_last::operator-=( - const years& y) NOEXCEPT { - *this = *this - y; - return *this; -} - -CONSTCD11 inline year year_month_weekday_last::year() const NOEXCEPT { return y_; } -CONSTCD11 inline month year_month_weekday_last::month() const NOEXCEPT { return m_; } - -CONSTCD11 -inline weekday year_month_weekday_last::weekday() const NOEXCEPT { - return wdl_.weekday(); -} - -CONSTCD11 -inline weekday_last year_month_weekday_last::weekday_last() const NOEXCEPT { - return wdl_; -} - -CONSTCD14 -inline year_month_weekday_last::operator sys_days() const NOEXCEPT { - return sys_days{to_days()}; -} - -CONSTCD14 -inline year_month_weekday_last::operator local_days() const NOEXCEPT { - return local_days{to_days()}; -} - -CONSTCD11 -inline bool year_month_weekday_last::ok() const NOEXCEPT { - return y_.ok() && m_.ok() && wdl_.ok(); -} - -CONSTCD14 -inline days year_month_weekday_last::to_days() const NOEXCEPT { - auto const d = sys_days(y_ / m_ / last); - return (d - (date::weekday{d} - wdl_.weekday())).time_since_epoch(); -} - -CONSTCD11 -inline bool operator==(const year_month_weekday_last& x, - const year_month_weekday_last& y) NOEXCEPT { - return x.year() == y.year() && x.month() == y.month() && - x.weekday_last() == y.weekday_last(); -} - -CONSTCD11 -inline bool operator!=(const year_month_weekday_last& x, - const year_month_weekday_last& y) NOEXCEPT { - return !(x == y); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const year_month_weekday_last& ymwdl) { - return os << ymwdl.year() << '/' << ymwdl.month() << '/' << ymwdl.weekday_last(); -} - -template -CONSTCD14 inline year_month_weekday_last operator+(const year_month_weekday_last& ymwdl, - const months& dm) NOEXCEPT { - return (ymwdl.year() / ymwdl.month() + dm) / ymwdl.weekday_last(); -} - -template -CONSTCD14 inline year_month_weekday_last operator+( - const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT { - return ymwdl + dm; -} - -template -CONSTCD14 inline year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, - const months& dm) NOEXCEPT { - return ymwdl + (-dm); -} - -CONSTCD11 -inline year_month_weekday_last operator+(const year_month_weekday_last& ymwdl, - const years& dy) NOEXCEPT { - return {ymwdl.year() + dy, ymwdl.month(), ymwdl.weekday_last()}; -} - -CONSTCD11 -inline year_month_weekday_last operator+(const years& dy, - const year_month_weekday_last& ymwdl) NOEXCEPT { - return ymwdl + dy; -} - -CONSTCD11 -inline year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, - const years& dy) NOEXCEPT { - return ymwdl + (-dy); -} - -// year_month from operator/() - -CONSTCD11 -inline year_month operator/(const year& y, const month& m) NOEXCEPT { return {y, m}; } - -CONSTCD11 -inline year_month operator/(const year& y, int m) NOEXCEPT { - return y / month(static_cast(m)); -} - -// month_day from operator/() - -CONSTCD11 -inline month_day operator/(const month& m, const day& d) NOEXCEPT { return {m, d}; } - -CONSTCD11 -inline month_day operator/(const day& d, const month& m) NOEXCEPT { return m / d; } - -CONSTCD11 -inline month_day operator/(const month& m, int d) NOEXCEPT { - return m / day(static_cast(d)); -} - -CONSTCD11 -inline month_day operator/(int m, const day& d) NOEXCEPT { - return month(static_cast(m)) / d; -} - -CONSTCD11 inline month_day operator/(const day& d, int m) NOEXCEPT { return m / d; } - -// month_day_last from operator/() - -CONSTCD11 -inline month_day_last operator/(const month& m, last_spec) NOEXCEPT { - return month_day_last{m}; -} - -CONSTCD11 -inline month_day_last operator/(last_spec, const month& m) NOEXCEPT { return m / last; } - -CONSTCD11 -inline month_day_last operator/(int m, last_spec) NOEXCEPT { - return month(static_cast(m)) / last; -} - -CONSTCD11 -inline month_day_last operator/(last_spec, int m) NOEXCEPT { return m / last; } - -// month_weekday from operator/() - -CONSTCD11 -inline month_weekday operator/(const month& m, const weekday_indexed& wdi) NOEXCEPT { - return {m, wdi}; -} - -CONSTCD11 -inline month_weekday operator/(const weekday_indexed& wdi, const month& m) NOEXCEPT { - return m / wdi; -} - -CONSTCD11 -inline month_weekday operator/(int m, const weekday_indexed& wdi) NOEXCEPT { - return month(static_cast(m)) / wdi; -} - -CONSTCD11 -inline month_weekday operator/(const weekday_indexed& wdi, int m) NOEXCEPT { - return m / wdi; -} - -// month_weekday_last from operator/() - -CONSTCD11 -inline month_weekday_last operator/(const month& m, const weekday_last& wdl) NOEXCEPT { - return {m, wdl}; -} - -CONSTCD11 -inline month_weekday_last operator/(const weekday_last& wdl, const month& m) NOEXCEPT { - return m / wdl; -} - -CONSTCD11 -inline month_weekday_last operator/(int m, const weekday_last& wdl) NOEXCEPT { - return month(static_cast(m)) / wdl; -} - -CONSTCD11 -inline month_weekday_last operator/(const weekday_last& wdl, int m) NOEXCEPT { - return m / wdl; -} - -// year_month_day from operator/() - -CONSTCD11 -inline year_month_day operator/(const year_month& ym, const day& d) NOEXCEPT { - return {ym.year(), ym.month(), d}; -} - -CONSTCD11 -inline year_month_day operator/(const year_month& ym, int d) NOEXCEPT { - return ym / day(static_cast(d)); -} - -CONSTCD11 -inline year_month_day operator/(const year& y, const month_day& md) NOEXCEPT { - return y / md.month() / md.day(); -} - -CONSTCD11 -inline year_month_day operator/(int y, const month_day& md) NOEXCEPT { - return year(y) / md; -} - -CONSTCD11 -inline year_month_day operator/(const month_day& md, const year& y) NOEXCEPT { - return y / md; -} - -CONSTCD11 -inline year_month_day operator/(const month_day& md, int y) NOEXCEPT { - return year(y) / md; -} - -// year_month_day_last from operator/() - -CONSTCD11 -inline year_month_day_last operator/(const year_month& ym, last_spec) NOEXCEPT { - return {ym.year(), month_day_last{ym.month()}}; -} - -CONSTCD11 -inline year_month_day_last operator/(const year& y, const month_day_last& mdl) NOEXCEPT { - return {y, mdl}; -} - -CONSTCD11 -inline year_month_day_last operator/(int y, const month_day_last& mdl) NOEXCEPT { - return year(y) / mdl; -} - -CONSTCD11 -inline year_month_day_last operator/(const month_day_last& mdl, const year& y) NOEXCEPT { - return y / mdl; -} - -CONSTCD11 -inline year_month_day_last operator/(const month_day_last& mdl, int y) NOEXCEPT { - return year(y) / mdl; -} - -// year_month_weekday from operator/() - -CONSTCD11 -inline year_month_weekday operator/(const year_month& ym, - const weekday_indexed& wdi) NOEXCEPT { - return {ym.year(), ym.month(), wdi}; -} - -CONSTCD11 -inline year_month_weekday operator/(const year& y, const month_weekday& mwd) NOEXCEPT { - return {y, mwd.month(), mwd.weekday_indexed()}; -} - -CONSTCD11 -inline year_month_weekday operator/(int y, const month_weekday& mwd) NOEXCEPT { - return year(y) / mwd; -} - -CONSTCD11 -inline year_month_weekday operator/(const month_weekday& mwd, const year& y) NOEXCEPT { - return y / mwd; -} - -CONSTCD11 -inline year_month_weekday operator/(const month_weekday& mwd, int y) NOEXCEPT { - return year(y) / mwd; -} - -// year_month_weekday_last from operator/() - -CONSTCD11 -inline year_month_weekday_last operator/(const year_month& ym, - const weekday_last& wdl) NOEXCEPT { - return {ym.year(), ym.month(), wdl}; -} - -CONSTCD11 -inline year_month_weekday_last operator/(const year& y, - const month_weekday_last& mwdl) NOEXCEPT { - return {y, mwdl.month(), mwdl.weekday_last()}; -} - -CONSTCD11 -inline year_month_weekday_last operator/(int y, const month_weekday_last& mwdl) NOEXCEPT { - return year(y) / mwdl; -} - -CONSTCD11 -inline year_month_weekday_last operator/(const month_weekday_last& mwdl, - const year& y) NOEXCEPT { - return y / mwdl; -} - -CONSTCD11 -inline year_month_weekday_last operator/(const month_weekday_last& mwdl, int y) NOEXCEPT { - return year(y) / mwdl; -} - -template -struct fields; - -template -std::basic_ostream& to_stream( - std::basic_ostream& os, const CharT* fmt, const fields& fds, - const std::string* abbrev = nullptr, - const std::chrono::seconds* offset_sec = nullptr); - -template -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, fields& fds, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr); - -// time_of_day - -enum { am = 1, pm }; - -namespace detail { - -// width::value is the number of fractional decimal digits in 1/n -// width<0>::value and width<1>::value are defined to be 0 -// If 1/n takes more than 18 fractional decimal digits, -// the result is truncated to 19. -// Example: width<2>::value == 1 -// Example: width<3>::value == 19 -// Example: width<4>::value == 2 -// Example: width<10>::value == 1 -// Example: width<1000>::value == 3 -template -struct width { - static CONSTDATA unsigned value = 1 + width::value; -}; - -template -struct width { - static CONSTDATA unsigned value = 0; -}; - -template -struct static_pow10 { - private: - static CONSTDATA std::uint64_t h = static_pow10::value; - - public: - static CONSTDATA std::uint64_t value = h * h * (exp % 2 ? 10 : 1); -}; - -template <> -struct static_pow10<0> { - static CONSTDATA std::uint64_t value = 1; -}; - -template -struct make_precision { - using type = std::chrono::duration::value>>; - static CONSTDATA unsigned width = w; -}; - -template -struct make_precision { - using type = std::chrono::duration; - static CONSTDATA unsigned width = 6; -}; - -template ::type::period::den>::value> -class decimal_format_seconds { - public: - using rep = typename std::common_type::type::rep; - using precision = typename make_precision::type; - static auto CONSTDATA width = make_precision::width; - - private: - std::chrono::seconds s_; - precision sub_s_; - - public: - CONSTCD11 decimal_format_seconds() : s_(), sub_s_() {} - - CONSTCD11 explicit decimal_format_seconds(const Duration& d) NOEXCEPT - : s_(std::chrono::duration_cast(d)), - sub_s_(std::chrono::duration_cast(d - s_)) {} - - CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT { return s_; } - CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT { return s_; } - CONSTCD11 precision subseconds() const NOEXCEPT { return sub_s_; } - - CONSTCD14 precision to_duration() const NOEXCEPT { return s_ + sub_s_; } - - CONSTCD11 bool in_conventional_range() const NOEXCEPT { - return sub_s_ < std::chrono::seconds{1} && s_ < std::chrono::minutes{1}; - } - - template - friend std::basic_ostream& operator<<( - std::basic_ostream& os, const decimal_format_seconds& x) { - date::detail::save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << x.s_.count() - << std::use_facet>(os.getloc()).decimal_point(); - os.width(width); - os << static_cast(x.sub_s_.count()); - return os; - } -}; - -template -class decimal_format_seconds { - static CONSTDATA unsigned w = 0; - - public: - using rep = typename std::common_type::type::rep; - using precision = std::chrono::duration; - static auto CONSTDATA width = make_precision::width; - - private: - std::chrono::seconds s_; - - public: - CONSTCD11 decimal_format_seconds() : s_() {} - CONSTCD11 explicit decimal_format_seconds(const precision& s) NOEXCEPT : s_(s) {} - - CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT { return s_; } - CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT { return s_; } - CONSTCD14 precision to_duration() const NOEXCEPT { return s_; } - - CONSTCD11 bool in_conventional_range() const NOEXCEPT { - return s_ < std::chrono::minutes{1}; - } - - template - friend std::basic_ostream& operator<<( - std::basic_ostream& os, const decimal_format_seconds& x) { - date::detail::save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << x.s_.count(); - return os; - } -}; - -enum class classify { not_valid, hour, minute, second, subsecond }; - -template -struct classify_duration { - static CONSTDATA classify value = - std::is_convertible::value - ? classify::hour - : std::is_convertible::value - ? classify::minute - : std::is_convertible::value - ? classify::second - : std::chrono::treat_as_floating_point< - typename Duration::rep>::value - ? classify::not_valid - : classify::subsecond; -}; - -template -inline CONSTCD11 typename std::enable_if::is_signed, - std::chrono::duration>::type -abs(std::chrono::duration d) { - return d >= d.zero() ? +d : -d; -} - -template -inline CONSTCD11 typename std::enable_if::is_signed, - std::chrono::duration>::type -abs(std::chrono::duration d) { - return d; -} - -class time_of_day_base { - protected: - std::chrono::hours h_; - unsigned char mode_; - bool neg_; - - enum { is24hr }; - - CONSTCD11 time_of_day_base() NOEXCEPT : h_(0), - mode_(static_cast(is24hr)), - neg_(false) {} - - CONSTCD11 time_of_day_base(std::chrono::hours h, bool neg, unsigned m) NOEXCEPT - : h_(detail::abs(h)), - mode_(static_cast(m)), - neg_(neg) {} - - CONSTCD14 void make24() NOEXCEPT; - CONSTCD14 void make12() NOEXCEPT; - - CONSTCD14 std::chrono::hours to24hr() const; - - CONSTCD11 bool in_conventional_range() const NOEXCEPT { return !neg_ && h_ < days{1}; } -}; - -CONSTCD14 -inline std::chrono::hours time_of_day_base::to24hr() const { - auto h = h_; - if (mode_ == am || mode_ == pm) { - CONSTDATA auto h12 = std::chrono::hours(12); - if (mode_ == pm) { - if (h != h12) h = h + h12; - } else if (h == h12) { - h = std::chrono::hours(0); - } - } - return h; -} - -CONSTCD14 -inline void time_of_day_base::make24() NOEXCEPT { - h_ = to24hr(); - mode_ = is24hr; -} - -CONSTCD14 -inline void time_of_day_base::make12() NOEXCEPT { - if (mode_ == is24hr) { - CONSTDATA auto h12 = std::chrono::hours(12); - if (h_ >= h12) { - if (h_ > h12) h_ = h_ - h12; - mode_ = pm; - } else { - if (h_ == std::chrono::hours(0)) h_ = h12; - mode_ = am; - } - } -} - -template ::value> -class time_of_day_storage; - -template -class time_of_day_storage, detail::classify::hour> - : private detail::time_of_day_base { - using base = detail::time_of_day_base; - - public: - using precision = std::chrono::hours; - -#if !defined(_MSC_VER) || _MSC_VER >= 1900 - CONSTCD11 time_of_day_storage() NOEXCEPT = default; -#else - CONSTCD11 time_of_day_storage() = default; -#endif /* !defined(_MSC_VER) || _MSC_VER >= 1900 */ - - CONSTCD11 explicit time_of_day_storage(std::chrono::hours since_midnight) NOEXCEPT - : base(since_midnight, since_midnight < std::chrono::hours{0}, is24hr) {} - - CONSTCD11 explicit time_of_day_storage(std::chrono::hours h, unsigned md) NOEXCEPT - : base(h, h < std::chrono::hours{0}, md) {} - - CONSTCD11 std::chrono::hours hours() const NOEXCEPT { return h_; } - CONSTCD11 unsigned mode() const NOEXCEPT { return mode_; } - - CONSTCD14 explicit operator precision() const NOEXCEPT { - auto p = to24hr(); - if (neg_) p = -p; - return p; - } - - CONSTCD14 precision to_duration() const NOEXCEPT { - return static_cast(*this); - } - - CONSTCD14 time_of_day_storage& make24() NOEXCEPT { - base::make24(); - return *this; - } - CONSTCD14 time_of_day_storage& make12() NOEXCEPT { - base::make12(); - return *this; - } - - CONSTCD11 bool in_conventional_range() const NOEXCEPT { - return base::in_conventional_range(); - } - - template - friend std::basic_ostream& operator<<( - std::basic_ostream& os, const time_of_day_storage& t) { - detail::save_ostream _(os); - if (t.neg_) os << '-'; - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - if (t.mode_ != am && t.mode_ != pm) os.width(2); - os << t.h_.count(); - switch (t.mode_) { - case time_of_day_storage::is24hr: - os << "00"; - break; - case am: - os << "am"; - break; - case pm: - os << "pm"; - break; - } - return os; - } -}; - -template -class time_of_day_storage, detail::classify::minute> - : private detail::time_of_day_base { - using base = detail::time_of_day_base; - - std::chrono::minutes m_; - - public: - using precision = std::chrono::minutes; - - CONSTCD11 time_of_day_storage() NOEXCEPT : base(), m_(0) {} - - CONSTCD11 explicit time_of_day_storage(std::chrono::minutes since_midnight) NOEXCEPT - : base(std::chrono::duration_cast(since_midnight), - since_midnight < std::chrono::minutes{0}, is24hr), - m_(detail::abs(since_midnight) - h_) {} - - CONSTCD11 explicit time_of_day_storage(std::chrono::hours h, std::chrono::minutes m, - unsigned md) NOEXCEPT : base(h, false, md), - m_(m) {} - - CONSTCD11 std::chrono::hours hours() const NOEXCEPT { return h_; } - CONSTCD11 std::chrono::minutes minutes() const NOEXCEPT { return m_; } - CONSTCD11 unsigned mode() const NOEXCEPT { return mode_; } - - CONSTCD14 explicit operator precision() const NOEXCEPT { - auto p = to24hr() + m_; - if (neg_) p = -p; - return p; - } - - CONSTCD14 precision to_duration() const NOEXCEPT { - return static_cast(*this); - } - - CONSTCD14 time_of_day_storage& make24() NOEXCEPT { - base::make24(); - return *this; - } - CONSTCD14 time_of_day_storage& make12() NOEXCEPT { - base::make12(); - return *this; - } - - CONSTCD11 bool in_conventional_range() const NOEXCEPT { - return base::in_conventional_range() && m_ < std::chrono::hours{1}; - } - - template - friend std::basic_ostream& operator<<( - std::basic_ostream& os, const time_of_day_storage& t) { - detail::save_ostream _(os); - if (t.neg_) os << '-'; - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - if (t.mode_ != am && t.mode_ != pm) os.width(2); - os << t.h_.count() << ':'; - os.width(2); - os << t.m_.count(); - switch (t.mode_) { - case am: - os << "am"; - break; - case pm: - os << "pm"; - break; - } - return os; - } -}; - -template -class time_of_day_storage, detail::classify::second> - : private detail::time_of_day_base { - using base = detail::time_of_day_base; - using dfs = decimal_format_seconds; - - std::chrono::minutes m_; - dfs s_; - - public: - using precision = std::chrono::seconds; - - CONSTCD11 time_of_day_storage() NOEXCEPT : base(), m_(0), s_() {} - - CONSTCD11 explicit time_of_day_storage(std::chrono::seconds since_midnight) NOEXCEPT - : base(std::chrono::duration_cast(since_midnight), - since_midnight < std::chrono::seconds{0}, is24hr), - m_(std::chrono::duration_cast(detail::abs(since_midnight) - - h_)), - s_(detail::abs(since_midnight) - h_ - m_) {} - - CONSTCD11 explicit time_of_day_storage(std::chrono::hours h, std::chrono::minutes m, - std::chrono::seconds s, unsigned md) NOEXCEPT - : base(h, false, md), - m_(m), - s_(s) {} - - CONSTCD11 std::chrono::hours hours() const NOEXCEPT { return h_; } - CONSTCD11 std::chrono::minutes minutes() const NOEXCEPT { return m_; } - CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT { return s_.seconds(); } - CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT { return s_.seconds(); } - CONSTCD11 unsigned mode() const NOEXCEPT { return mode_; } - - CONSTCD14 explicit operator precision() const NOEXCEPT { - auto p = to24hr() + s_.to_duration() + m_; - if (neg_) p = -p; - return p; - } - - CONSTCD14 precision to_duration() const NOEXCEPT { - return static_cast(*this); - } - - CONSTCD14 time_of_day_storage& make24() NOEXCEPT { - base::make24(); - return *this; - } - CONSTCD14 time_of_day_storage& make12() NOEXCEPT { - base::make12(); - return *this; - } - - CONSTCD11 bool in_conventional_range() const NOEXCEPT { - return base::in_conventional_range() && m_ < std::chrono::hours{1} && - s_.in_conventional_range(); - } - - template - friend std::basic_ostream& operator<<( - std::basic_ostream& os, const time_of_day_storage& t) { - detail::save_ostream _(os); - if (t.neg_) os << '-'; - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - if (t.mode_ != am && t.mode_ != pm) os.width(2); - os << t.h_.count() << ':'; - os.width(2); - os << t.m_.count() << ':' << t.s_; - switch (t.mode_) { - case am: - os << "am"; - break; - case pm: - os << "pm"; - break; - } - return os; - } - - template - friend std::basic_ostream& date::to_stream( - std::basic_ostream& os, const CharT* fmt, - const fields& fds, const std::string* abbrev, - const std::chrono::seconds* offset_sec); - - template - friend std::basic_istream& date::from_stream( - std::basic_istream& is, const CharT* fmt, fields& fds, - std::basic_string* abbrev, std::chrono::minutes* offset); -}; - -template -class time_of_day_storage, detail::classify::subsecond> - : private detail::time_of_day_base { - public: - using Duration = std::chrono::duration; - using dfs = decimal_format_seconds< - typename std::common_type::type>; - using precision = typename dfs::precision; - - private: - using base = detail::time_of_day_base; - - std::chrono::minutes m_; - dfs s_; - - public: - CONSTCD11 time_of_day_storage() NOEXCEPT : base(), m_(0), s_() {} - - CONSTCD11 explicit time_of_day_storage(Duration since_midnight) NOEXCEPT - : base(date::trunc(since_midnight), - since_midnight < Duration{0}, is24hr), - m_(date::trunc(detail::abs(since_midnight) - h_)), - s_(detail::abs(since_midnight) - h_ - m_) {} - - CONSTCD11 explicit time_of_day_storage(std::chrono::hours h, std::chrono::minutes m, - std::chrono::seconds s, precision sub_s, - unsigned md) NOEXCEPT : base(h, false, md), - m_(m), - s_(s + sub_s) {} - - CONSTCD11 std::chrono::hours hours() const NOEXCEPT { return h_; } - CONSTCD11 std::chrono::minutes minutes() const NOEXCEPT { return m_; } - CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT { return s_.seconds(); } - CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT { return s_.seconds(); } - CONSTCD11 precision subseconds() const NOEXCEPT { return s_.subseconds(); } - CONSTCD11 unsigned mode() const NOEXCEPT { return mode_; } - - CONSTCD14 explicit operator precision() const NOEXCEPT { - auto p = to24hr() + s_.to_duration() + m_; - if (neg_) p = -p; - return p; - } - - CONSTCD14 precision to_duration() const NOEXCEPT { - return static_cast(*this); - } - - CONSTCD14 time_of_day_storage& make24() NOEXCEPT { - base::make24(); - return *this; - } - CONSTCD14 time_of_day_storage& make12() NOEXCEPT { - base::make12(); - return *this; - } - - CONSTCD11 bool in_conventional_range() const NOEXCEPT { - return base::in_conventional_range() && m_ < std::chrono::hours{1} && - s_.in_conventional_range(); - } - - template - friend std::basic_ostream& operator<<( - std::basic_ostream& os, const time_of_day_storage& t) { - detail::save_ostream _(os); - if (t.neg_) os << '-'; - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - if (t.mode_ != am && t.mode_ != pm) os.width(2); - os << t.h_.count() << ':'; - os.width(2); - os << t.m_.count() << ':' << t.s_; - switch (t.mode_) { - case am: - os << "am"; - break; - case pm: - os << "pm"; - break; - } - return os; - } - - template - friend std::basic_ostream& date::to_stream( - std::basic_ostream& os, const CharT* fmt, - const fields& fds, const std::string* abbrev, - const std::chrono::seconds* offset_sec); - - template - friend std::basic_istream& date::from_stream( - std::basic_istream& is, const CharT* fmt, fields& fds, - std::basic_string* abbrev, std::chrono::minutes* offset); -}; - -} // namespace detail - -template -class time_of_day : public detail::time_of_day_storage { - using base = detail::time_of_day_storage; - - public: -#if !defined(_MSC_VER) || _MSC_VER >= 1900 - CONSTCD11 time_of_day() NOEXCEPT = default; -#else - CONSTCD11 time_of_day() = default; -#endif /* !defined(_MSC_VER) || _MSC_VER >= 1900 */ - - CONSTCD11 explicit time_of_day(Duration since_midnight) NOEXCEPT - : base(since_midnight) {} - - template - CONSTCD11 explicit time_of_day(Arg0&& arg0, Arg1&& arg1, Args&&... args) NOEXCEPT - : base(std::forward(arg0), std::forward(arg1), - std::forward(args)...) {} -}; - -template ::value>::type> -CONSTCD11 inline time_of_day> make_time( - const std::chrono::duration& d) { - return time_of_day>(d); -} - -CONSTCD11 -inline time_of_day make_time(const std::chrono::hours& h, - unsigned md) { - return time_of_day(h, md); -} - -CONSTCD11 -inline time_of_day make_time(const std::chrono::hours& h, - const std::chrono::minutes& m, - unsigned md) { - return time_of_day(h, m, md); -} - -CONSTCD11 -inline time_of_day make_time(const std::chrono::hours& h, - const std::chrono::minutes& m, - const std::chrono::seconds& s, - unsigned md) { - return time_of_day(h, m, s, md); -} - -template < - class Rep, class Period, - class = typename std::enable_if>::value>::type> -CONSTCD11 inline time_of_day> make_time( - const std::chrono::hours& h, const std::chrono::minutes& m, - const std::chrono::seconds& s, const std::chrono::duration& sub_s, - unsigned md) { - return time_of_day>(h, m, s, sub_s, md); -} - -template -inline typename std::enable_if< - !std::chrono::treat_as_floating_point::value && - std::ratio_less::value, - std::basic_ostream&>::type -operator<<(std::basic_ostream& os, const sys_time& tp) { - auto const dp = date::floor(tp); - return os << year_month_day(dp) << ' ' << make_time(tp - dp); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const sys_days& dp) { - return os << year_month_day(dp); -} - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const local_time& ut) { - return (os << sys_time{ut.time_since_epoch()}); -} - -// to_stream - -CONSTDATA year nanyear{-32768}; - -template -struct fields { - year_month_day ymd{nanyear / 0 / 0}; - weekday wd{8u}; - time_of_day tod{}; - bool has_tod = false; - - fields() = default; - - explicit fields(year_month_day ymd_) : ymd(ymd_) {} - explicit fields(weekday wd_) : wd(wd_) {} - explicit fields(time_of_day tod_) : tod(tod_), has_tod(true) {} - - fields(year_month_day ymd_, weekday wd_) : ymd(ymd_), wd(wd_) {} - fields(year_month_day ymd_, time_of_day tod_) - : ymd(ymd_), tod(tod_), has_tod(true) {} - - fields(weekday wd_, time_of_day tod_) : wd(wd_), tod(tod_), has_tod(true) {} - - fields(year_month_day ymd_, weekday wd_, time_of_day tod_) - : ymd(ymd_), wd(wd_), tod(tod_), has_tod(true) {} -}; - -namespace detail { - -template -unsigned extract_weekday(std::basic_ostream& os, - const fields& fds) { - if (!fds.ymd.ok() && !fds.wd.ok()) { - // fds does not contain a valid weekday - os.setstate(std::ios::failbit); - return 8; - } - weekday wd; - if (fds.ymd.ok()) { - wd = weekday{sys_days(fds.ymd)}; - if (fds.wd.ok() && wd != fds.wd) { - // fds.ymd and fds.wd are inconsistent - os.setstate(std::ios::failbit); - return 8; - } - } else { - wd = fds.wd; - } - return static_cast((wd - Sunday).count()); -} - -template -unsigned extract_month(std::basic_ostream& os, - const fields& fds) { - if (!fds.ymd.month().ok()) { - // fds does not contain a valid month - os.setstate(std::ios::failbit); - return 0; - } - return static_cast(fds.ymd.month()); -} - -} // namespace detail - -#if ONLY_C_LOCALE - -namespace detail { - -inline std::pair weekday_names() { - static const string nm[] = {"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", - "Friday", "Saturday", "Sun", "Mon", "Tue", - "Wed", "Thu", "Fri", "Sat"}; - return make_pair(nm, nm + sizeof(nm) / sizeof(nm[0])); -} - -inline std::pair month_names() { - static const string nm[] = { - "January", "February", "March", "April", "May", "June", "July", "August", - "September", "October", "November", "December", "Jan", "Feb", "Mar", "Apr", - "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; - return make_pair(nm, nm + sizeof(nm) / sizeof(nm[0])); -} - -inline std::pair ampm_names() { - static const string nm[] = {"AM", "PM"}; - return make_pair(nm, nm + sizeof(nm) / sizeof(nm[0])); -} - -template -FwdIter scan_keyword(std::basic_istream& is, FwdIter kb, FwdIter ke) { - size_t nkw = static_cast(std::distance(kb, ke)); - const unsigned char doesnt_match = '\0'; - const unsigned char might_match = '\1'; - const unsigned char does_match = '\2'; - unsigned char statbuf[100]; - unsigned char* status = statbuf; - unique_ptr stat_hold(0, free); - if (nkw > sizeof(statbuf)) { - status = (unsigned char*)malloc(nkw); - if (status == nullptr) throw bad_alloc(); - stat_hold.reset(status); - } - size_t n_might_match = nkw; // At this point, any keyword might match - size_t n_does_match = 0; // but none of them definitely do - // Initialize all statuses to might_match, except for "" keywords are does_match - unsigned char* st = status; - for (auto ky = kb; ky != ke; ++ky, ++st) { - if (!ky->empty()) { - *st = might_match; - } else { - *st = does_match; - --n_might_match; - ++n_does_match; - } - } - // While there might be a match, test keywords against the next CharT - for (size_t indx = 0; is && n_might_match > 0; ++indx) { - // Peek at the next CharT but don't consume it - auto ic = is.peek(); - if (ic == EOF) { - is.setstate(ios::eofbit); - break; - } - auto c = static_cast(toupper(ic)); - bool consume = false; - // For each keyword which might match, see if the indx character is c - // If a match if found, consume c - // If a match is found, and that is the last character in the keyword, - // then that keyword matches. - // If the keyword doesn't match this character, then change the keyword - // to doesn't match - st = status; - for (auto ky = kb; ky != ke; ++ky, ++st) { - if (*st == might_match) { - if (c == static_cast(toupper((*ky)[indx]))) { - consume = true; - if (ky->size() == indx + 1) { - *st = does_match; - --n_might_match; - ++n_does_match; - } - } else { - *st = doesnt_match; - --n_might_match; - } - } - } - // consume if we matched a character - if (consume) { - (void)is.get(); - // If we consumed a character and there might be a matched keyword that - // was marked matched on a previous iteration, then such keywords - // are now marked as not matching. - if (n_might_match + n_does_match > 1) { - st = status; - for (auto ky = kb; ky != ke; ++ky, ++st) { - if (*st == does_match && ky->size() != indx + 1) { - *st = doesnt_match; - --n_does_match; - } - } - } - } - } - // We've exited the loop because we hit eof and/or we have no more "might matches". - // Return the first matching result - for (st = status; kb != ke; ++kb, ++st) - if (*st == does_match) break; - if (kb == ke) is.setstate(ios_base::failbit); - return kb; -} - -} // namespace detail - -#endif // ONLY_C_LOCALE - -template -std::basic_ostream& to_stream(std::basic_ostream& os, - const CharT* fmt, - const fields& fds, - const std::string* abbrev, - const std::chrono::seconds* offset_sec) { - using detail::save_ostream; - using std::ios; - using std::time_put; - using std::use_facet; - using std::chrono::duration; - using std::chrono::duration_cast; - using std::chrono::hours; - using std::chrono::minutes; - date::detail::save_ostream ss(os); - os.fill(' '); - os.flags(std::ios::skipws | std::ios::dec); - os.width(0); - tm tm{}; - bool insert_negative = fds.has_tod && fds.tod.to_duration() < Duration::zero(); -#if !ONLY_C_LOCALE - auto& facet = use_facet>(os.getloc()); -#endif - const CharT* command = nullptr; - CharT modified = CharT{}; - for (; *fmt; ++fmt) { - switch (*fmt) { - case 'a': - case 'A': - if (command) { - if (modified == CharT{}) { - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) return os; -#if !ONLY_C_LOCALE - const CharT f[] = {'%', *fmt}; - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); -#else // ONLY_C_LOCALE - os << weekday_names().first[tm.tm_wday + 7 * (*fmt == 'a')]; -#endif // ONLY_C_LOCALE - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'b': - case 'B': - case 'h': - if (command) { - if (modified == CharT{}) { - tm.tm_mon = static_cast(extract_month(os, fds)) - 1; -#if !ONLY_C_LOCALE - const CharT f[] = {'%', *fmt}; - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); -#else // ONLY_C_LOCALE - os << month_names().first[tm.tm_mon + 12 * (*fmt != 'B')]; -#endif // ONLY_C_LOCALE - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'c': - case 'x': - if (command) { - if (modified == CharT{'O'}) { - os << CharT{'%'} << modified << *fmt; - } else { - if (!fds.ymd.ok()) os.setstate(std::ios::failbit); - if (*fmt == 'c' && !fds.has_tod) os.setstate(std::ios::failbit); -#if !ONLY_C_LOCALE - tm = std::tm{}; - auto const& ymd = fds.ymd; - auto ld = local_days(ymd); - if (*fmt == 'c') { - tm.tm_sec = static_cast(fds.tod.seconds().count()); - tm.tm_min = static_cast(fds.tod.minutes().count()); - tm.tm_hour = static_cast(fds.tod.hours().count()); - } - tm.tm_mday = static_cast(static_cast(ymd.day())); - tm.tm_mon = static_cast(extract_month(os, fds) - 1); - tm.tm_year = static_cast(ymd.year()) - 1900; - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) return os; - tm.tm_yday = static_cast((ld - local_days(ymd.year() / 1 / 1)).count()); - CharT f[3] = {'%'}; - auto fe = begin(f) + 1; - if (modified == CharT{'E'}) *fe++ = modified; - *fe++ = *fmt; - facet.put(os, os, os.fill(), &tm, begin(f), fe); -#else // ONLY_C_LOCALE - if (*fmt == 'c') { - auto wd = static_cast(extract_weekday(os, fds)); - os << weekday_names().first[static_cast(wd) + 7] << ' '; - os << month_names().first[extract_month(os, fds) - 1 + 12] << ' '; - auto d = static_cast(static_cast(fds.ymd.day())); - if (d < 10) { - os << ' '; - } - os << d << ' ' << make_time(duration_cast(fds.tod.to_duration())) - << ' ' << fds.ymd.year(); - } else { // *fmt == 'x' - auto const& ymd = fds.ymd; - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << static_cast(ymd.month()) << CharT{'/'}; - os.width(2); - os << static_cast(ymd.day()) << CharT{'/'}; - os.width(2); - os << static_cast(ymd.year()) % 100; - } -#endif // ONLY_C_LOCALE - } - command = nullptr; - modified = CharT{}; - } else { - os << *fmt; - } - break; - case 'C': - if (command) { - if (modified == CharT{'O'}) { - os << CharT{'%'} << modified << *fmt; - } else { - if (!fds.ymd.year().ok()) os.setstate(std::ios::failbit); - auto y = static_cast(fds.ymd.year()); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - if (y >= 0) { - os.width(2); - os << y / 100; - } else { - os << CharT{'-'}; - os.width(2); - os << -(y - 99) / 100; - } - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'E'}) { - tm.tm_year = y - 1900; - CharT f[3] = {'%', 'E', 'C'}; - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - command = nullptr; - modified = CharT{}; - } else { - os << *fmt; - } - break; - case 'd': - case 'e': - if (command) { - if (modified == CharT{'E'}) { - os << CharT{'%'} << modified << *fmt; - } else { - if (!fds.ymd.day().ok()) os.setstate(std::ios::failbit); - auto d = static_cast(static_cast(fds.ymd.day())); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - save_ostream _(os); - if (*fmt == CharT{'d'}) { - os.fill('0'); - } else { - os.fill(' '); - } - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << d; - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - tm.tm_mday = d; - CharT f[3] = {'%', 'O', *fmt}; - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - command = nullptr; - modified = CharT{}; - } else { - os << *fmt; - } - break; - case 'D': - if (command) { - if (modified == CharT{}) { - if (!fds.ymd.ok()) { - os.setstate(std::ios::failbit); - } - auto const& ymd = fds.ymd; - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << static_cast(ymd.month()) << CharT{'/'}; - os.width(2); - os << static_cast(ymd.day()) << CharT{'/'}; - os.width(2); - os << static_cast(ymd.year()) % 100; - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'F': - if (command) { - if (modified == CharT{}) { - if (!fds.ymd.ok()) { - os.setstate(std::ios::failbit); - } - auto const& ymd = fds.ymd; - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(4); - os << static_cast(ymd.year()) << CharT{'-'}; - os.width(2); - os << static_cast(ymd.month()) << CharT{'-'}; - os.width(2); - os << static_cast(ymd.day()); - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'g': - case 'G': - if (command) { - if (modified == CharT{}) { - if (!fds.ymd.ok()) os.setstate(std::ios::failbit); - auto ld = local_days(fds.ymd); - auto y = year_month_day{ld + days{3}}.year(); - auto start = local_days((y - years{1}) / December / Thursday[last]) + - (Monday - Thursday); - if (ld < start) { - --y; - } - if (*fmt == CharT{'G'}) { - os << y; - } else { - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(2); - os << std::abs(static_cast(y)) % 100; - } - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'H': - case 'I': - if (command) { - if (modified == CharT{'E'}) { - os << CharT{'%'} << modified << *fmt; - } else { - if (!fds.has_tod) { - os.setstate(std::ios::failbit); - } - if (insert_negative) { - os << '-'; - insert_negative = false; - } - auto hms = fds.tod; -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - if (*fmt == CharT{'I'}) hms.make12(); - if (hms.hours() < hours{10}) os << CharT{'0'}; - os << hms.hours().count(); - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_hour = static_cast(hms.hours().count()); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 'j': - if (command) { - if (modified == CharT{}) { - if (!fds.ymd.ok()) { - os.setstate(std::ios::failbit); - } - auto ld = local_days(fds.ymd); - auto y = fds.ymd.year(); - auto doy = ld - local_days(y / January / 1) + days{1}; - save_ostream _(os); - os.fill('0'); - os.flags(std::ios::dec | std::ios::right); - os.width(3); - os << doy.count(); - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'm': - if (command) { - if (modified == CharT{'E'}) { - os << CharT{'%'} << modified << *fmt; - } else { - if (!fds.ymd.month().ok()) os.setstate(std::ios::failbit); - auto m = static_cast(fds.ymd.month()); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - if (m < 10) os << CharT{'0'}; - os << m; - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_mon = static_cast(m - 1); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 'M': - if (command) { - if (modified == CharT{'E'}) { - os << CharT{'%'} << modified << *fmt; - } else { - if (!fds.has_tod) os.setstate(std::ios::failbit); - if (insert_negative) { - os << '-'; - insert_negative = false; - } -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - if (fds.tod.minutes() < minutes{10}) os << CharT{'0'}; - os << fds.tod.minutes().count(); - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_min = static_cast(fds.tod.minutes().count()); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 'n': - if (command) { - if (modified == CharT{}) - os << CharT{'\n'}; - else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'p': - if (command) { - if (modified == CharT{}) { - if (!fds.has_tod) os.setstate(std::ios::failbit); -#if !ONLY_C_LOCALE - const CharT f[] = {'%', *fmt}; - tm.tm_hour = static_cast(fds.tod.hours().count()); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); -#else - if (fds.tod.hours() < hours{12}) { - os << ampm_names().first[0]; - } else { - os << ampm_names().first[1]; - } -#endif - } else { - os << CharT{'%'} << modified << *fmt; - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 'r': - if (command) { - if (modified == CharT{}) { - if (!fds.has_tod) os.setstate(std::ios::failbit); -#if !ONLY_C_LOCALE - const CharT f[] = {'%', *fmt}; - tm.tm_hour = static_cast(fds.tod.hours().count()); - tm.tm_min = static_cast(fds.tod.minutes().count()); - tm.tm_sec = static_cast(fds.tod.seconds().count()); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); -#else - time_of_day tod(duration_cast(fds.tod.to_duration())); - tod.make12(); - save_ostream _(os); - os.fill('0'); - os.width(2); - os << tod.hours().count() << CharT{':'}; - os.width(2); - os << tod.minutes().count() << CharT{':'}; - os.width(2); - os << tod.seconds().count() << CharT{' '}; - tod.make24(); - if (tod.hours() < hours{12}) { - os << ampm_names().first[0]; - } else { - os << ampm_names().first[1]; - } -#endif - } else { - os << CharT{'%'} << modified << *fmt; - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 'R': - if (command) { - if (modified == CharT{}) { - if (!fds.has_tod) { - os.setstate(std::ios::failbit); - } - if (fds.tod.hours() < hours{10}) { - os << CharT{'0'}; - } - os << fds.tod.hours().count() << CharT{':'}; - if (fds.tod.minutes() < minutes{10}) { - os << CharT{'0'}; - } - os << fds.tod.minutes().count(); - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'S': - if (command) { - if (modified == CharT{'E'}) { - os << CharT{'%'} << modified << *fmt; - } else { - if (!fds.has_tod) os.setstate(std::ios::failbit); - if (insert_negative) { - os << '-'; - insert_negative = false; - } -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - os << fds.tod.s_; - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_sec = static_cast(fds.tod.s_.seconds().count()); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 't': - if (command) { - if (modified == CharT{}) { - os << CharT{'\t'}; - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'T': - if (command) { - if (modified == CharT{}) { - if (!fds.has_tod) os.setstate(std::ios::failbit); - os << fds.tod; - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else { - os << *fmt; - } - break; - case 'u': - if (command) { - if (modified == CharT{'E'}) { - os << CharT{'%'} << modified << *fmt; - } else { - auto wd = extract_weekday(os, fds); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - os << (wd != 0 ? wd : 7u); - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_wday = static_cast(wd); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 'U': - if (command) { - if (modified == CharT{'E'}) { - os << CharT{'%'} << modified << *fmt; - } else { - auto const& ymd = fds.ymd; - if (!ymd.ok()) os.setstate(std::ios::failbit); - auto ld = local_days(ymd); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - auto st = local_days(Sunday[1] / January / ymd.year()); - if (ld < st) - os << CharT{'0'} << CharT{'0'}; - else { - auto wn = duration_cast(ld - st).count() + 1; - if (wn < 10) os << CharT{'0'}; - os << wn; - } - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_year = static_cast(ymd.year()) - 1900; - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) return os; - tm.tm_yday = - static_cast((ld - local_days(ymd.year() / 1 / 1)).count()); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 'V': - if (command) { - if (modified == CharT{'E'}) { - os << CharT{'%'} << modified << *fmt; - } else { - if (!fds.ymd.ok()) os.setstate(std::ios::failbit); - auto ld = local_days(fds.ymd); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - auto y = year_month_day{ld + days{3}}.year(); - auto st = - local_days((y - years{1}) / 12 / Thursday[last]) + (Monday - Thursday); - if (ld < st) { - --y; - st = local_days((y - years{1}) / 12 / Thursday[last]) + - (Monday - Thursday); - } - auto wn = duration_cast(ld - st).count() + 1; - if (wn < 10) os << CharT{'0'}; - os << wn; - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - const CharT f[] = {'%', modified, *fmt}; - auto const& ymd = fds.ymd; - tm.tm_year = static_cast(ymd.year()) - 1900; - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) return os; - tm.tm_yday = - static_cast((ld - local_days(ymd.year() / 1 / 1)).count()); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 'w': - if (command) { - auto wd = extract_weekday(os, fds); - if (os.fail()) return os; -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#else - if (modified != CharT{'E'}) -#endif - { - os << wd; - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_wday = static_cast(wd); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - else { - os << CharT{'%'} << modified << *fmt; - } - modified = CharT{}; - command = nullptr; - } else { - os << *fmt; - } - break; - case 'W': - if (command) { - if (modified == CharT{'E'}) { - os << CharT{'%'} << modified << *fmt; - } else { - auto const& ymd = fds.ymd; - if (!ymd.ok()) os.setstate(std::ios::failbit); - auto ld = local_days(ymd); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - auto st = local_days(Monday[1] / January / ymd.year()); - if (ld < st) - os << CharT{'0'} << CharT{'0'}; - else { - auto wn = duration_cast(ld - st).count() + 1; - if (wn < 10) os << CharT{'0'}; - os << wn; - } - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_year = static_cast(ymd.year()) - 1900; - tm.tm_wday = static_cast(extract_weekday(os, fds)); - if (os.fail()) return os; - tm.tm_yday = - static_cast((ld - local_days(ymd.year() / 1 / 1)).count()); - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; - } else - os << *fmt; - break; - case 'X': - if (command) { - if (modified == CharT{'O'}) - os << CharT{'%'} << modified << *fmt; - else { - if (!fds.has_tod) os.setstate(std::ios::failbit); -#if !ONLY_C_LOCALE - tm = std::tm{}; - tm.tm_sec = static_cast(fds.tod.seconds().count()); - tm.tm_min = static_cast(fds.tod.minutes().count()); - tm.tm_hour = static_cast(fds.tod.hours().count()); - CharT f[3] = {'%'}; - auto fe = begin(f) + 1; - if (modified == CharT{'E'}) *fe++ = modified; - *fe++ = *fmt; - facet.put(os, os, os.fill(), &tm, begin(f), fe); -#else - os << fds.tod; -#endif - } - command = nullptr; - modified = CharT{}; - } else - os << *fmt; - break; - case 'y': - if (command) { - if (!fds.ymd.year().ok()) os.setstate(std::ios::failbit); - auto y = static_cast(fds.ymd.year()); -#if !ONLY_C_LOCALE - if (modified == CharT{}) { -#endif - y = std::abs(y) % 100; - if (y < 10) os << CharT{'0'}; - os << y; -#if !ONLY_C_LOCALE - } else { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_year = y - 1900; - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - modified = CharT{}; - command = nullptr; - } else - os << *fmt; - break; - case 'Y': - if (command) { - if (modified == CharT{'O'}) - os << CharT{'%'} << modified << *fmt; - else { - if (!fds.ymd.year().ok()) os.setstate(std::ios::failbit); - auto y = fds.ymd.year(); -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - os << y; - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'E'}) { - const CharT f[] = {'%', modified, *fmt}; - tm.tm_year = static_cast(y) - 1900; - facet.put(os, os, os.fill(), &tm, begin(f), end(f)); - } -#endif - } - modified = CharT{}; - command = nullptr; - } else - os << *fmt; - break; - case 'z': - if (command) { - if (offset_sec == nullptr) { - // Can not format %z with unknown offset - os.setstate(ios::failbit); - return os; - } - auto m = duration_cast(*offset_sec); - auto neg = m < minutes{0}; - m = date::abs(m); - auto h = duration_cast(m); - m -= h; - if (neg) - os << CharT{'-'}; - else - os << CharT{'+'}; - if (h < hours{10}) os << CharT{'0'}; - os << h.count(); - if (modified != CharT{}) os << CharT{':'}; - if (m < minutes{10}) os << CharT{'0'}; - os << m.count(); - command = nullptr; - modified = CharT{}; - } else - os << *fmt; - break; - case 'Z': - if (command) { - if (modified == CharT{}) { - if (abbrev == nullptr) { - // Can not format %Z with unknown time_zone - os.setstate(ios::failbit); - return os; - } - for (auto c : *abbrev) os << CharT(c); - } else { - os << CharT{'%'} << modified << *fmt; - modified = CharT{}; - } - command = nullptr; - } else - os << *fmt; - break; - case 'E': - case 'O': - if (command) { - if (modified == CharT{}) { - modified = *fmt; - } else { - os << CharT{'%'} << modified << *fmt; - command = nullptr; - modified = CharT{}; - } - } else - os << *fmt; - break; - case '%': - if (command) { - if (modified == CharT{}) { - os << CharT{'%'}; - command = nullptr; - } else { - os << CharT{'%'} << modified << CharT{'%'}; - command = nullptr; - modified = CharT{}; - } - } else - command = fmt; - break; - default: - if (command) { - os << CharT{'%'}; - command = nullptr; - } - if (modified != CharT{}) { - os << modified; - modified = CharT{}; - } - os << *fmt; - break; - } - } - if (command) os << CharT{'%'}; - if (modified != CharT{}) os << modified; - return os; -} - -template -inline std::basic_ostream& to_stream(std::basic_ostream& os, - const CharT* fmt, const year& y) { - using CT = std::chrono::seconds; - fields fds{y / 0 / 0}; - return to_stream(os, fmt, fds); -} - -template -inline std::basic_ostream& to_stream(std::basic_ostream& os, - const CharT* fmt, const month& m) { - using CT = std::chrono::seconds; - fields fds{m / 0 / nanyear}; - return to_stream(os, fmt, fds); -} - -template -inline std::basic_ostream& to_stream(std::basic_ostream& os, - const CharT* fmt, const day& d) { - using CT = std::chrono::seconds; - fields fds{d / 0 / nanyear}; - return to_stream(os, fmt, fds); -} - -template -inline std::basic_ostream& to_stream(std::basic_ostream& os, - const CharT* fmt, const weekday& wd) { - using CT = std::chrono::seconds; - fields fds{wd}; - return to_stream(os, fmt, fds); -} - -template -inline std::basic_ostream& to_stream(std::basic_ostream& os, - const CharT* fmt, - const year_month& ym) { - using CT = std::chrono::seconds; - fields fds{ym / 0}; - return to_stream(os, fmt, fds); -} - -template -inline std::basic_ostream& to_stream(std::basic_ostream& os, - const CharT* fmt, - const month_day& md) { - using CT = std::chrono::seconds; - fields fds{md / nanyear}; - return to_stream(os, fmt, fds); -} - -template -inline std::basic_ostream& to_stream(std::basic_ostream& os, - const CharT* fmt, - const year_month_day& ymd) { - using CT = std::chrono::seconds; - fields fds{ymd}; - return to_stream(os, fmt, fds); -} - -template -inline std::basic_ostream& to_stream( - std::basic_ostream& os, const CharT* fmt, - const std::chrono::duration& d) { - using Duration = std::chrono::duration; - using CT = typename std::common_type::type; - fields fds{time_of_day{d}}; - return to_stream(os, fmt, fds); -} - -template -std::basic_ostream& to_stream( - std::basic_ostream& os, const CharT* fmt, - const local_time& tp, const std::string* abbrev = nullptr, - const std::chrono::seconds* offset_sec = nullptr) { - using CT = typename std::common_type::type; - auto ld = floor(tp); - fields fds{year_month_day{ld}, time_of_day{tp - local_seconds{ld}}}; - return to_stream(os, fmt, fds, abbrev, offset_sec); -} - -template -std::basic_ostream& to_stream(std::basic_ostream& os, - const CharT* fmt, - const sys_time& tp) { - using namespace std::chrono; - using CT = typename std::common_type::type; - const std::string abbrev("UTC"); - CONSTDATA seconds offset{0}; - auto sd = floor(tp); - fields fds{year_month_day{sd}, time_of_day{tp - sys_seconds{sd}}}; - return to_stream(os, fmt, fds, &abbrev, &offset); -} - -// format - -template -auto format(const std::locale& loc, const CharT* fmt, const Streamable& tp) - -> decltype(to_stream(std::declval&>(), fmt, tp), - std::basic_string{}) { - std::basic_ostringstream os; - os.exceptions(std::ios::failbit | std::ios::badbit); - os.imbue(loc); - to_stream(os, fmt, tp); - return os.str(); -} - -template -auto format(const CharT* fmt, const Streamable& tp) - -> decltype(to_stream(std::declval&>(), fmt, tp), - std::basic_string{}) { - std::basic_ostringstream os; - os.exceptions(std::ios::failbit | std::ios::badbit); - to_stream(os, fmt, tp); - return os.str(); -} - -template -auto format(const std::locale& loc, const std::basic_string& fmt, - const Streamable& tp) - -> decltype(to_stream(std::declval&>(), fmt.c_str(), - tp), - std::basic_string{}) { - std::basic_ostringstream os; - os.exceptions(std::ios::failbit | std::ios::badbit); - os.imbue(loc); - to_stream(os, fmt.c_str(), tp); - return os.str(); -} - -template -auto format(const std::basic_string& fmt, const Streamable& tp) - -> decltype(to_stream(std::declval&>(), fmt.c_str(), - tp), - std::basic_string{}) { - std::basic_ostringstream os; - os.exceptions(std::ios::failbit | std::ios::badbit); - to_stream(os, fmt.c_str(), tp); - return os.str(); -} - -// parse - -namespace detail { - -template -bool read_char(std::basic_istream& is, CharT fmt, std::ios::iostate& err) { - auto ic = is.get(); - if (Traits::eq_int_type(ic, Traits::eof()) || - !Traits::eq(Traits::to_char_type(ic), fmt)) { - err |= std::ios::failbit; - is.setstate(std::ios::failbit); - return false; - } - return true; -} - -template -unsigned read_unsigned(std::basic_istream& is, unsigned m = 1, - unsigned M = 10) { - unsigned x = 0; - unsigned count = 0; - while (true) { - auto ic = is.peek(); - if (Traits::eq_int_type(ic, Traits::eof())) break; - auto c = static_cast(Traits::to_char_type(ic)); - if (!('0' <= c && c <= '9')) break; - (void)is.get(); - ++count; - x = 10 * x + static_cast(c - '0'); - if (count == M) break; - } - if (count < m) is.setstate(std::ios::failbit); - return x; -} - -template -int read_signed(std::basic_istream& is, unsigned m = 1, unsigned M = 10) { - auto ic = is.peek(); - if (!Traits::eq_int_type(ic, Traits::eof())) { - auto c = static_cast(Traits::to_char_type(ic)); - if (('0' <= c && c <= '9') || c == '-' || c == '+') { - if (c == '-' || c == '+') (void)is.get(); - auto x = static_cast(read_unsigned(is, std::max(m, 1u), M)); - if (!is.fail()) { - if (c == '-') x = -x; - return x; - } - } - } - if (m > 0) is.setstate(std::ios::failbit); - return 0; -} - -template -long double read_long_double(std::basic_istream& is, unsigned m = 1, - unsigned M = 10) { - using namespace std; - unsigned count = 0; - auto decimal_point = - Traits::to_int_type(use_facet>(is.getloc()).decimal_point()); - std::string buf; - while (true) { - auto ic = is.peek(); - if (Traits::eq_int_type(ic, Traits::eof())) break; - if (Traits::eq_int_type(ic, decimal_point)) { - buf += '.'; - decimal_point = Traits::eof(); - is.get(); - } else { - auto c = static_cast(Traits::to_char_type(ic)); - if (!('0' <= c && c <= '9')) break; - buf += c; - (void)is.get(); - } - if (++count == M) break; - } - if (count < m) { - is.setstate(std::ios::failbit); - return 0; - } - return std::stold(buf); -} - -struct rs { - int& i; - unsigned m; - unsigned M; -}; - -struct ru { - int& i; - unsigned m; - unsigned M; -}; - -struct rld { - long double& i; - unsigned m; - unsigned M; -}; - -template -void read(std::basic_istream&) {} - -template -void read(std::basic_istream& is, CharT a0, Args&&... args); - -template -void read(std::basic_istream& is, rs a0, Args&&... args); - -template -void read(std::basic_istream& is, ru a0, Args&&... args); - -template -void read(std::basic_istream& is, int a0, Args&&... args); - -template -void read(std::basic_istream& is, rld a0, Args&&... args); - -template -void read(std::basic_istream& is, CharT a0, Args&&... args) { - // No-op if a0 == CharT{} - if (a0 != CharT{}) { - auto ic = is.peek(); - if (Traits::eq_int_type(ic, Traits::eof())) { - is.setstate(std::ios::failbit | std::ios::eofbit); - return; - } - if (!Traits::eq(Traits::to_char_type(ic), a0)) { - is.setstate(std::ios::failbit); - return; - } - (void)is.get(); - } - read(is, std::forward(args)...); -} - -template -void read(std::basic_istream& is, rs a0, Args&&... args) { - auto x = read_signed(is, a0.m, a0.M); - if (is.fail()) return; - a0.i = x; - read(is, std::forward(args)...); -} - -template -void read(std::basic_istream& is, ru a0, Args&&... args) { - auto x = read_unsigned(is, a0.m, a0.M); - if (is.fail()) return; - a0.i = static_cast(x); - read(is, std::forward(args)...); -} - -template -void read(std::basic_istream& is, int a0, Args&&... args) { - if (a0 != -1) { - auto u = static_cast(a0); - CharT buf[std::numeric_limits::digits10 + 2] = {}; - auto e = buf; - do { - *e++ = CharT(u % 10) + CharT{'0'}; - u /= 10; - } while (u > 0); - std::reverse(buf, e); - for (auto p = buf; p != e && is.rdstate() == std::ios::goodbit; ++p) read(is, *p); - } - if (is.rdstate() == std::ios::goodbit) read(is, std::forward(args)...); -} - -template -void read(std::basic_istream& is, rld a0, Args&&... args) { - auto x = read_long_double(is, a0.m, a0.M); - if (is.fail()) return; - a0.i = x; - read(is, std::forward(args)...); -} - -template -inline void checked_set(T& value, T from, T not_a_value, - std::basic_ios& is) { - if (!is.fail()) { - if (value == not_a_value) - value = std::move(from); - else if (value != from) - is.setstate(std::ios::failbit); - } -} - -} // namespace detail - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, fields& fds, - std::basic_string* abbrev, std::chrono::minutes* offset) { - using namespace std; - using namespace std::chrono; - typename basic_istream::sentry ok{is, true}; - if (ok) { - date::detail::save_istream ss(is); - is.fill(' '); - is.flags(std::ios::skipws | std::ios::dec); - is.width(0); -#if !ONLY_C_LOCALE - auto& f = use_facet>(is.getloc()); - std::tm tm{}; -#endif - const CharT* command = nullptr; - auto modified = CharT{}; - auto width = -1; - - CONSTDATA int not_a_year = numeric_limits::min(); - CONSTDATA int not_a_2digit_year = 100; - CONSTDATA int not_a_century = not_a_year / 100; - CONSTDATA int not_a_month = 0; - CONSTDATA int not_a_day = 0; - CONSTDATA int not_a_hour = numeric_limits::min(); - CONSTDATA int not_a_hour_12_value = 0; - CONSTDATA int not_a_minute = not_a_hour; - CONSTDATA Duration not_a_second = Duration::min(); - CONSTDATA int not_a_doy = 0; - CONSTDATA int not_a_weekday = 8; - CONSTDATA int not_a_week_num = 100; - CONSTDATA int not_a_ampm = -1; - CONSTDATA minutes not_a_offset = minutes::min(); - - int Y = not_a_year; // c, F, Y * - int y = not_a_2digit_year; // D, x, y * - int g = not_a_2digit_year; // g * - int G = not_a_year; // G * - int C = not_a_century; // C * - int m = not_a_month; // b, B, h, m, c, D, F, x * - int d = not_a_day; // c, d, D, e, F, x * - int j = not_a_doy; // j * - int wd = not_a_weekday; // a, A, u, w * - int H = not_a_hour; // c, H, R, T, X * - int I = not_a_hour_12_value; // I, r * - int p = not_a_ampm; // p, r * - int M = not_a_minute; // c, M, r, R, T, X * - Duration s = not_a_second; // c, r, S, T, X * - int U = not_a_week_num; // U * - int V = not_a_week_num; // V * - int W = not_a_week_num; // W * - std::basic_string temp_abbrev; // Z * - minutes temp_offset = not_a_offset; // z * - - using detail::checked_set; - using detail::read; - using detail::rld; - using detail::rs; - using detail::ru; - for (; *fmt && is.rdstate() == std::ios::goodbit; ++fmt) { - switch (*fmt) { - case 'a': - case 'A': - case 'u': - case 'w': // wd: a, A, u, w - if (command) { - int trial_wd = not_a_weekday; - if (*fmt == 'a' || *fmt == 'A') { - if (modified == CharT{}) { -#if !ONLY_C_LOCALE - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - is.setstate(err); - if (!is.fail()) trial_wd = tm.tm_wday; -#else - auto nm = detail::weekday_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - if (!is.fail()) trial_wd = i % 7; -#endif - } else - read(is, CharT{'%'}, width, modified, *fmt); - } else // *fmt == 'u' || *fmt == 'w' - { -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#else - if (modified != CharT{'E'}) -#endif - { - read(is, - ru{trial_wd, 1, width == -1 ? 1u : static_cast(width)}); - if (!is.fail()) { - if (*fmt == 'u') { - if (!(1 <= trial_wd && trial_wd <= 7)) { - trial_wd = not_a_weekday; - is.setstate(ios_base::failbit); - } else if (trial_wd == 7) - trial_wd = 0; - } else // *fmt == 'w' - { - if (!(0 <= trial_wd && trial_wd <= 6)) { - trial_wd = not_a_weekday; - is.setstate(ios_base::failbit); - } - } - } - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - is.setstate(err); - if (!is.fail()) trial_wd = tm.tm_wday; - } -#endif - else - read(is, CharT{'%'}, width, modified, *fmt); - } - if (trial_wd != not_a_weekday) checked_set(wd, trial_wd, not_a_weekday, is); - } else // !command - read(is, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - break; - case 'b': - case 'B': - case 'h': - if (command) { - if (modified == CharT{}) { - int ttm = not_a_month; -#if !ONLY_C_LOCALE - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) ttm = tm.tm_mon + 1; - is.setstate(err); -#else - auto nm = detail::month_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - if (!is.fail()) ttm = i % 12 + 1; -#endif - checked_set(m, ttm, not_a_month, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'c': - if (command) { - if (modified != CharT{'O'}) { -#if !ONLY_C_LOCALE - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) { - checked_set(Y, tm.tm_year + 1900, not_a_year, is); - checked_set(m, tm.tm_mon + 1, not_a_month, is); - checked_set(d, tm.tm_mday, not_a_day, is); - checked_set(H, tm.tm_hour, not_a_hour, is); - checked_set(M, tm.tm_min, not_a_minute, is); - checked_set(s, duration_cast(seconds{tm.tm_sec}), not_a_second, - is); - } - is.setstate(err); -#else - // "%a %b %e %T %Y" - auto nm = detail::weekday_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - checked_set(wd, static_cast(i % 7), not_a_weekday, is); - ws(is); - nm = detail::month_names(); - i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - checked_set(m, static_cast(i % 12 + 1), not_a_month, is); - ws(is); - int td = not_a_day; - read(is, rs{td, 1, 2}); - checked_set(d, td, not_a_day, is); - ws(is); - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - int tH; - int tM; - long double S; - read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, CharT{':'}, rld{S, 1, w}); - checked_set(H, tH, not_a_hour, is); - checked_set(M, tM, not_a_minute, is); - checked_set(s, round(duration{S}), not_a_second, is); - ws(is); - int tY = not_a_year; - read(is, rs{tY, 1, 4u}); - checked_set(Y, tY, not_a_year, is); -#endif - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'x': - if (command) { - if (modified != CharT{'O'}) { -#if !ONLY_C_LOCALE - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) { - checked_set(Y, tm.tm_year + 1900, not_a_year, is); - checked_set(m, tm.tm_mon + 1, not_a_month, is); - checked_set(d, tm.tm_mday, not_a_day, is); - } - is.setstate(err); -#else - // "%m/%d/%y" - int ty = not_a_2digit_year; - int tm = not_a_month; - int td = not_a_day; - read(is, ru{tm, 1, 2}, CharT{'/'}, ru{td, 1, 2}, CharT{'/'}, rs{ty, 1, 2}); - checked_set(y, ty, not_a_2digit_year, is); - checked_set(m, tm, not_a_month, is); - checked_set(d, td, not_a_day, is); -#endif - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'X': - if (command) { - if (modified != CharT{'O'}) { -#if !ONLY_C_LOCALE - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) { - checked_set(H, tm.tm_hour, not_a_hour, is); - checked_set(M, tm.tm_min, not_a_minute, is); - checked_set(s, duration_cast(seconds{tm.tm_sec}), not_a_second, - is); - } - is.setstate(err); -#else - // "%T" - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - int tH = not_a_hour; - int tM = not_a_minute; - long double S; - read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, CharT{':'}, rld{S, 1, w}); - checked_set(H, tH, not_a_hour, is); - checked_set(M, tM, not_a_minute, is); - checked_set(s, round(duration{S}), not_a_second, is); -#endif - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'C': - if (command) { - int tC = not_a_century; -#if !ONLY_C_LOCALE - if (modified == CharT{}) { -#endif - read(is, rs{tC, 1, width == -1 ? 2u : static_cast(width)}); -#if !ONLY_C_LOCALE - } else { - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) { - auto tY = tm.tm_year + 1900; - tC = (tY >= 0 ? tY : tY - 99) / 100; - } - is.setstate(err); - } -#endif - checked_set(C, tC, not_a_century, is); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'D': - if (command) { - if (modified == CharT{}) { - int tn = not_a_month; - int td = not_a_day; - int ty = not_a_2digit_year; - read(is, ru{tn, 1, 2}, CharT{'\0'}, CharT{'/'}, CharT{'\0'}, ru{td, 1, 2}, - CharT{'\0'}, CharT{'/'}, CharT{'\0'}, rs{ty, 1, 2}); - checked_set(y, ty, not_a_2digit_year, is); - checked_set(m, tn, not_a_month, is); - checked_set(d, td, not_a_day, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'F': - if (command) { - if (modified == CharT{}) { - int tY = not_a_year; - int tn = not_a_month; - int td = not_a_day; - read(is, rs{tY, 1, width == -1 ? 4u : static_cast(width)}, - CharT{'-'}, ru{tn, 1, 2}, CharT{'-'}, ru{td, 1, 2}); - checked_set(Y, tY, not_a_year, is); - checked_set(m, tn, not_a_month, is); - checked_set(d, td, not_a_day, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'd': - case 'e': - if (command) { -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#else - if (modified != CharT{'E'}) -#endif - { - int td = not_a_day; - read(is, rs{td, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(d, td, not_a_day, is); - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - command = nullptr; - width = -1; - modified = CharT{}; - if ((err & ios::failbit) == 0) checked_set(d, tm.tm_mday, not_a_day, is); - is.setstate(err); - } -#endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'H': - if (command) { -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#else - if (modified != CharT{'E'}) -#endif - { - int tH = not_a_hour; - read(is, ru{tH, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(H, tH, not_a_hour, is); - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) checked_set(H, tm.tm_hour, not_a_hour, is); - is.setstate(err); - } -#endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'I': - if (command) { - if (modified == CharT{}) { - int tI = not_a_hour_12_value; - // reads in an hour into I, but most be in [1, 12] - read(is, rs{tI, 1, width == -1 ? 2u : static_cast(width)}); - if (!(1 <= tI && tI <= 12)) is.setstate(ios::failbit); - checked_set(I, tI, not_a_hour_12_value, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'j': - if (command) { - if (modified == CharT{}) { - int tj = not_a_doy; - read(is, ru{tj, 1, width == -1 ? 3u : static_cast(width)}); - checked_set(j, tj, not_a_doy, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'M': - if (command) { -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#else - if (modified != CharT{'E'}) -#endif - { - int tM = not_a_minute; - read(is, ru{tM, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(M, tM, not_a_minute, is); - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) checked_set(M, tm.tm_min, not_a_minute, is); - is.setstate(err); - } -#endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'm': - if (command) { -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#else - if (modified != CharT{'E'}) -#endif - { - int tn = not_a_month; - read(is, rs{tn, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(m, tn, not_a_month, is); - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) - checked_set(m, tm.tm_mon + 1, not_a_month, is); - is.setstate(err); - } -#endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'n': - case 't': - if (command) { - if (modified == CharT{}) { - // %n matches a single white space character - // %t matches 0 or 1 white space characters - auto ic = is.peek(); - if (Traits::eq_int_type(ic, Traits::eof())) { - ios_base::iostate err = ios_base::eofbit; - if (*fmt == 'n') err |= ios_base::failbit; - is.setstate(err); - break; - } - if (isspace(ic)) { - (void)is.get(); - } else if (*fmt == 'n') - is.setstate(ios_base::failbit); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'p': - if (command) { - if (modified == CharT{}) { - int tp = not_a_ampm; -#if !ONLY_C_LOCALE - tm = std::tm{}; - tm.tm_hour = 1; - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - is.setstate(err); - if (tm.tm_hour == 1) - tp = 0; - else if (tm.tm_hour == 13) - tp = 1; - else - is.setstate(err); -#else - auto nm = detail::ampm_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - tp = i; -#endif - checked_set(p, tp, not_a_ampm, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - - break; - case 'r': - if (command) { - if (modified == CharT{}) { -#if !ONLY_C_LOCALE - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) { - checked_set(H, tm.tm_hour, not_a_hour, is); - checked_set(M, tm.tm_min, not_a_hour, is); - checked_set(s, duration_cast(seconds{tm.tm_sec}), not_a_second, - is); - } - is.setstate(err); -#else - // "%I:%M:%S %p" - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - long double S; - int tI = not_a_hour_12_value; - int tM = not_a_minute; - read(is, ru{tI, 1, 2}, CharT{':'}, ru{tM, 1, 2}, CharT{':'}, rld{S, 1, w}); - checked_set(I, tI, not_a_hour_12_value, is); - checked_set(M, tM, not_a_minute, is); - checked_set(s, round(duration{S}), not_a_second, is); - ws(is); - auto nm = detail::ampm_names(); - auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; - checked_set(p, static_cast(i), not_a_ampm, is); -#endif - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'R': - if (command) { - if (modified == CharT{}) { - int tH = not_a_hour; - int tM = not_a_minute; - read(is, ru{tH, 1, 2}, CharT{'\0'}, CharT{':'}, CharT{'\0'}, ru{tM, 1, 2}, - CharT{'\0'}); - checked_set(H, tH, not_a_hour, is); - checked_set(M, tM, not_a_minute, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'S': - if (command) { -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#else - if (modified != CharT{'E'}) -#endif - { - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - long double S; - read(is, rld{S, 1, width == -1 ? w : static_cast(width)}); - checked_set(s, round(duration{S}), not_a_second, is); - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'O'}) { - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) - checked_set(s, duration_cast(seconds{tm.tm_sec}), not_a_second, - is); - is.setstate(err); - } -#endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'T': - if (command) { - if (modified == CharT{}) { - using dfs = detail::decimal_format_seconds; - CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; - int tH = not_a_hour; - int tM = not_a_minute; - long double S; - read(is, ru{tH, 1, 2}, CharT{':'}, ru{tM, 1, 2}, CharT{':'}, rld{S, 1, w}); - checked_set(H, tH, not_a_hour, is); - checked_set(M, tM, not_a_minute, is); - checked_set(s, round(duration{S}), not_a_second, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'Y': - if (command) { -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#else - if (modified != CharT{'O'}) -#endif - { - int tY = not_a_year; - read(is, rs{tY, 1, width == -1 ? 4u : static_cast(width)}); - checked_set(Y, tY, not_a_year, is); - } -#if !ONLY_C_LOCALE - else if (modified == CharT{'E'}) { - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) - checked_set(Y, tm.tm_year + 1900, not_a_year, is); - is.setstate(err); - } -#endif - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'y': - if (command) { -#if !ONLY_C_LOCALE - if (modified == CharT{}) -#endif - { - int ty = not_a_2digit_year; - read(is, ru{ty, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(y, ty, not_a_2digit_year, is); - } -#if !ONLY_C_LOCALE - else { - ios_base::iostate err = ios_base::goodbit; - f.get(is, nullptr, is, err, &tm, command, fmt + 1); - if ((err & ios::failbit) == 0) - checked_set(Y, tm.tm_year + 1900, not_a_year, is); - is.setstate(err); - } -#endif - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'g': - if (command) { - if (modified == CharT{}) { - int tg = not_a_2digit_year; - read(is, ru{tg, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(g, tg, not_a_2digit_year, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'G': - if (command) { - if (modified == CharT{}) { - int tG = not_a_year; - read(is, rs{tG, 1, width == -1 ? 4u : static_cast(width)}); - checked_set(G, tG, not_a_year, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'U': - if (command) { - if (modified == CharT{}) { - int tU = not_a_week_num; - read(is, ru{tU, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(U, tU, not_a_week_num, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'V': - if (command) { - if (modified == CharT{}) { - int tV = not_a_week_num; - read(is, ru{tV, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(V, tV, not_a_week_num, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'W': - if (command) { - if (modified == CharT{}) { - int tW = not_a_week_num; - read(is, ru{tW, 1, width == -1 ? 2u : static_cast(width)}); - checked_set(W, tW, not_a_week_num, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'E': - case 'O': - if (command) { - if (modified == CharT{}) { - modified = *fmt; - } else { - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - } else - read(is, *fmt); - break; - case '%': - if (command) { - if (modified == CharT{}) - read(is, *fmt); - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - command = fmt; - break; - case 'z': - if (command) { - int tH, tM; - minutes toff = not_a_offset; - bool neg = false; - auto ic = is.peek(); - if (!Traits::eq_int_type(ic, Traits::eof())) { - auto c = static_cast(Traits::to_char_type(ic)); - if (c == '-') neg = true; - } - if (modified == CharT{}) { - read(is, rs{tH, 2, 2}); - if (!is.fail()) toff = hours{std::abs(tH)}; - if (is.good()) { - ic = is.peek(); - if (!Traits::eq_int_type(ic, Traits::eof())) { - auto c = static_cast(Traits::to_char_type(ic)); - if ('0' <= c && c <= '9') { - read(is, ru{tM, 2, 2}); - if (!is.fail()) toff += minutes{tM}; - } - } - } - } else { - read(is, rs{tH, 1, 2}); - if (!is.fail()) toff = hours{std::abs(tH)}; - if (is.good()) { - ic = is.peek(); - if (!Traits::eq_int_type(ic, Traits::eof())) { - auto c = static_cast(Traits::to_char_type(ic)); - if (c == ':') { - (void)is.get(); - read(is, ru{tM, 2, 2}); - if (!is.fail()) toff += minutes{tM}; - } - } - } - } - if (neg) toff = -toff; - checked_set(temp_offset, toff, not_a_offset, is); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - case 'Z': - if (command) { - if (modified == CharT{}) { - std::basic_string buf; - while (is.rdstate() == std::ios::goodbit) { - auto i = is.rdbuf()->sgetc(); - if (Traits::eq_int_type(i, Traits::eof())) { - is.setstate(ios::eofbit); - break; - } - auto wc = Traits::to_char_type(i); - auto c = static_cast(wc); - // is c a valid time zone name or abbreviation character? - if (!(CharT{1} < wc && wc < CharT{127}) || - !(isalnum(c) || c == '_' || c == '/' || c == '-' || c == '+')) - break; - buf.push_back(c); - is.rdbuf()->sbumpc(); - } - if (buf.empty()) is.setstate(ios::failbit); - checked_set(temp_abbrev, buf, {}, is); - } else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } else - read(is, *fmt); - break; - default: - if (command) { - if (width == -1 && modified == CharT{} && '0' <= *fmt && *fmt <= '9') { - width = static_cast(*fmt) - '0'; - while ('0' <= fmt[1] && fmt[1] <= '9') - width = 10 * width + static_cast(*++fmt) - '0'; - } else { - if (modified == CharT{}) - read(is, CharT{'%'}, width, *fmt); - else - read(is, CharT{'%'}, width, modified, *fmt); - command = nullptr; - width = -1; - modified = CharT{}; - } - } else // !command - { - if (isspace(static_cast(*fmt))) - ws(is); // space matches 0 or more white space characters - else - read(is, *fmt); - } - break; - } - } - // is.rdstate() != ios::goodbit || *fmt == CharT{} - if (is.rdstate() == ios::goodbit && command) { - if (modified == CharT{}) - read(is, CharT{'%'}, width); - else - read(is, CharT{'%'}, width, modified); - } - if (is.rdstate() != ios::goodbit && *fmt != CharT{} && !is.fail()) - is.setstate(ios::failbit); - if (!is.fail()) { - if (y != not_a_2digit_year) { - // Convert y and an optional C to Y - if (!(0 <= y && y <= 99)) goto broken; - if (C == not_a_century) { - if (Y == not_a_year) { - if (y >= 69) - C = 19; - else - C = 20; - } else { - C = (Y >= 0 ? Y : Y - 100) / 100; - } - } - int tY; - if (C >= 0) - tY = 100 * C + y; - else - tY = 100 * (C + 1) - (y == 0 ? 100 : y); - if (Y != not_a_year && Y != tY) goto broken; - Y = tY; - } - if (g != not_a_2digit_year) { - // Convert g and an optional C to G - if (!(0 <= g && g <= 99)) goto broken; - if (C == not_a_century) { - if (G == not_a_year) { - if (g >= 69) - C = 19; - else - C = 20; - } else { - C = (G >= 0 ? G : G - 100) / 100; - } - } - int tG; - if (C >= 0) - tG = 100 * C + g; - else - tG = 100 * (C + 1) - (g == 0 ? 100 : g); - if (G != not_a_year && G != tG) goto broken; - G = tG; - } - if (Y < static_cast(year::min()) || Y > static_cast(year::max())) - Y = not_a_year; - bool computed = false; - if (G != not_a_year && V != not_a_week_num && wd != not_a_weekday) { - year_month_day ymd_trial = sys_days(year{G - 1} / December / Thursday[last]) + - (Monday - Thursday) + weeks{V - 1} + - (weekday{static_cast(wd)} - Monday); - if (Y == not_a_year) - Y = static_cast(ymd_trial.year()); - else if (year{Y} != ymd_trial.year()) - goto broken; - if (m == not_a_month) - m = static_cast(static_cast(ymd_trial.month())); - else if (month(static_cast(m)) != ymd_trial.month()) - goto broken; - if (d == not_a_day) - d = static_cast(static_cast(ymd_trial.day())); - else if (day(static_cast(d)) != ymd_trial.day()) - goto broken; - computed = true; - } - if (Y != not_a_year && U != not_a_week_num && wd != not_a_weekday) { - year_month_day ymd_trial = sys_days(year{Y} / January / Sunday[1]) + - weeks{U - 1} + - (weekday{static_cast(wd)} - Sunday); - if (Y == not_a_year) - Y = static_cast(ymd_trial.year()); - else if (year{Y} != ymd_trial.year()) - goto broken; - if (m == not_a_month) - m = static_cast(static_cast(ymd_trial.month())); - else if (month(static_cast(m)) != ymd_trial.month()) - goto broken; - if (d == not_a_day) - d = static_cast(static_cast(ymd_trial.day())); - else if (day(static_cast(d)) != ymd_trial.day()) - goto broken; - computed = true; - } - if (Y != not_a_year && W != not_a_week_num && wd != not_a_weekday) { - year_month_day ymd_trial = sys_days(year{Y} / January / Monday[1]) + - weeks{W - 1} + - (weekday{static_cast(wd)} - Monday); - if (Y == not_a_year) - Y = static_cast(ymd_trial.year()); - else if (year{Y} != ymd_trial.year()) - goto broken; - if (m == not_a_month) - m = static_cast(static_cast(ymd_trial.month())); - else if (month(static_cast(m)) != ymd_trial.month()) - goto broken; - if (d == not_a_day) - d = static_cast(static_cast(ymd_trial.day())); - else if (day(static_cast(d)) != ymd_trial.day()) - goto broken; - computed = true; - } - if (j != 0 && Y != not_a_year) { - auto ymd_trial = year_month_day{local_days(year{Y} / 1 / 1) + days{j - 1}}; - if (m == 0) - m = static_cast(static_cast(ymd_trial.month())); - else if (month(static_cast(m)) != ymd_trial.month()) - goto broken; - if (d == 0) - d = static_cast(static_cast(ymd_trial.day())); - else if (day(static_cast(d)) != ymd_trial.day()) - goto broken; - } - auto ymd = year{Y} / m / d; - if (ymd.ok()) { - if (wd == not_a_weekday) - wd = static_cast((weekday(sys_days(ymd)) - Sunday).count()); - else if (wd != static_cast((weekday(sys_days(ymd)) - Sunday).count())) - goto broken; - if (!computed) { - if (G != not_a_year || V != not_a_week_num) { - sys_days sd = ymd; - auto G_trial = year_month_day{sd + days{3}}.year(); - auto start = sys_days((G_trial - years{1}) / December / Thursday[last]) + - (Monday - Thursday); - if (sd < start) { - --G_trial; - if (V != not_a_week_num) - start = sys_days((G_trial - years{1}) / December / Thursday[last]) + - (Monday - Thursday); - } - if (G != not_a_year && G != static_cast(G_trial)) goto broken; - if (V != not_a_week_num) { - auto V_trial = duration_cast(sd - start).count() + 1; - if (V != V_trial) goto broken; - } - } - if (U != not_a_week_num) { - auto start = sys_days(Sunday[1] / January / ymd.year()); - auto U_trial = floor(sys_days(ymd) - start).count() + 1; - if (U != U_trial) goto broken; - } - if (W != not_a_week_num) { - auto start = sys_days(Monday[1] / January / ymd.year()); - auto W_trial = floor(sys_days(ymd) - start).count() + 1; - if (W != W_trial) goto broken; - } - } - } - fds.ymd = ymd; - if (I != not_a_hour_12_value) { - if (!(1 <= I && I <= 12)) goto broken; - if (p != not_a_ampm) { - // p is in [0, 1] == [AM, PM] - // Store trial H in I - if (I == 12) --p; - I += p * 12; - // Either set H from I or make sure H and I are consistent - if (H == not_a_hour) - H = I; - else if (I != H) - goto broken; - } else // p == not_a_ampm - { - // if H, make sure H and I could be consistent - if (H != not_a_hour) { - if (I == 12) { - if (H != 0 && H != 12) goto broken; - } else if (!(I == H || I == H + 12)) { - goto broken; - } - } - } - } - if (H != not_a_hour) { - fds.has_tod = true; - fds.tod = time_of_day{hours{H}}; - } - if (M != not_a_minute) { - fds.has_tod = true; - fds.tod.m_ = minutes{M}; - } - if (s != not_a_second) { - fds.has_tod = true; - fds.tod.s_ = detail::decimal_format_seconds{s}; - } - if (wd != not_a_weekday) fds.wd = weekday{static_cast(wd)}; - if (abbrev != nullptr) *abbrev = std::move(temp_abbrev); - if (offset != nullptr && temp_offset != not_a_offset) *offset = temp_offset; - } - return is; - } -broken: - is.setstate(ios_base::failbit); - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, year& y, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using CT = seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.year().ok()) is.setstate(ios::failbit); - if (!is.fail()) y = fds.ymd.year(); - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, month& m, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using CT = seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.month().ok()) is.setstate(ios::failbit); - if (!is.fail()) m = fds.ymd.month(); - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, day& d, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using CT = seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.day().ok()) is.setstate(ios::failbit); - if (!is.fail()) d = fds.ymd.day(); - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, weekday& wd, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using CT = seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.wd.ok()) is.setstate(ios::failbit); - if (!is.fail()) wd = fds.wd; - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, year_month& ym, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using CT = seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.month().ok()) is.setstate(ios::failbit); - if (!is.fail()) ym = fds.ymd.year() / fds.ymd.month(); - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, month_day& md, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using CT = seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.month().ok() || !fds.ymd.day().ok()) is.setstate(ios::failbit); - if (!is.fail()) md = fds.ymd.month() / fds.ymd.day(); - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, year_month_day& ymd, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using CT = seconds; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.ok()) is.setstate(ios::failbit); - if (!is.fail()) ymd = fds.ymd; - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, sys_time& tp, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using CT = typename common_type::type; - minutes offset_local{}; - auto offptr = offset ? offset : &offset_local; - fields fds{}; - fds.has_tod = true; - from_stream(is, fmt, fds, abbrev, offptr); - if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) is.setstate(ios::failbit); - if (!is.fail()) - tp = round(sys_days(fds.ymd) - *offptr + fds.tod.to_duration()); - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, local_time& tp, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using CT = typename common_type::type; - fields fds{}; - fds.has_tod = true; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) is.setstate(ios::failbit); - if (!is.fail()) - tp = round(local_seconds{local_days(fds.ymd)} + fds.tod.to_duration()); - return is; -} - -template > -std::basic_istream& from_stream( - std::basic_istream& is, const CharT* fmt, - std::chrono::duration& d, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) { - using namespace std; - using namespace std::chrono; - using Duration = std::chrono::duration; - using CT = typename common_type::type; - fields fds{}; - from_stream(is, fmt, fds, abbrev, offset); - if (!fds.has_tod) is.setstate(ios::failbit); - if (!is.fail()) d = duration_cast(fds.tod.to_duration()); - return is; -} - -template , - class Alloc = std::allocator> -struct parse_manip { - const std::basic_string format_; - Parsable& tp_; - std::basic_string* abbrev_; - std::chrono::minutes* offset_; - - public: - parse_manip(std::basic_string format, Parsable& tp, - std::basic_string* abbrev = nullptr, - std::chrono::minutes* offset = nullptr) - : format_(std::move(format)), tp_(tp), abbrev_(abbrev), offset_(offset) {} -}; - -template -std::basic_istream& operator>>( - std::basic_istream& is, - const parse_manip& x) { - return from_stream(is, x.format_.c_str(), x.tp_, x.abbrev_, x.offset_); -} - -template -inline auto parse(const std::basic_string& format, Parsable& tp) - -> decltype(from_stream(std::declval&>(), - format.c_str(), tp), - parse_manip{format, tp}) { - return {format, tp}; -} - -template -inline auto parse(const std::basic_string& format, Parsable& tp, - std::basic_string& abbrev) - -> decltype(from_stream(std::declval&>(), - format.c_str(), tp, &abbrev), - parse_manip{format, tp, &abbrev}) { - return {format, tp, &abbrev}; -} - -template -inline auto parse(const std::basic_string& format, Parsable& tp, - std::chrono::minutes& offset) - -> decltype(from_stream(std::declval&>(), - format.c_str(), tp, - std::declval*>(), - &offset), - parse_manip{format, tp, nullptr, - &offset}) { - return {format, tp, nullptr, &offset}; -} - -template -inline auto parse(const std::basic_string& format, Parsable& tp, - std::basic_string& abbrev, - std::chrono::minutes& offset) - -> decltype(from_stream(std::declval&>(), - format.c_str(), tp, &abbrev, &offset), - parse_manip{format, tp, &abbrev, - &offset}) { - return {format, tp, &abbrev, &offset}; -} - -// const CharT* formats - -template -inline auto parse(const CharT* format, Parsable& tp) - -> decltype(from_stream(std::declval&>(), format, tp), - parse_manip{format, tp}) { - return {format, tp}; -} - -template -inline auto parse(const CharT* format, Parsable& tp, - std::basic_string& abbrev) - -> decltype(from_stream(std::declval&>(), format, - tp, &abbrev), - parse_manip{format, tp, &abbrev}) { - return {format, tp, &abbrev}; -} - -template -inline auto parse(const CharT* format, Parsable& tp, std::chrono::minutes& offset) - -> decltype(from_stream(std::declval&>(), format, tp, - std::declval*>(), &offset), - parse_manip{format, tp, nullptr, &offset}) { - return {format, tp, nullptr, &offset}; -} - -template -inline auto parse(const CharT* format, Parsable& tp, - std::basic_string& abbrev, - std::chrono::minutes& offset) - -> decltype(from_stream(std::declval&>(), format, - tp, &abbrev, &offset), - parse_manip{format, tp, &abbrev, - &offset}) { - return {format, tp, &abbrev, &offset}; -} - -// duration streaming - -namespace detail { - -template -class string_literal; - -template -inline CONSTCD14 string_literal< - typename std::conditional::type, - N1 + N2 - 1> -operator+(const string_literal& x, - const string_literal& y) NOEXCEPT; - -template -class string_literal { - CharT p_[N]; - - CONSTCD11 string_literal() NOEXCEPT : p_{} {} - - public: - using const_iterator = const CharT*; - - string_literal(string_literal const&) = default; - string_literal& operator=(string_literal const&) = delete; - - template ::type> - CONSTCD11 string_literal(CharT c) NOEXCEPT : p_{c} {} - - template ::type> - CONSTCD11 string_literal(CharT c1, CharT c2) NOEXCEPT : p_{c1, c2} {} - - template ::type> - CONSTCD11 string_literal(CharT c1, CharT c2, CharT c3) NOEXCEPT : p_{c1, c2, c3} {} - - CONSTCD14 string_literal(const CharT (&a)[N]) NOEXCEPT : p_{} { - for (std::size_t i = 0; i < N; ++i) p_[i] = a[i]; - } - - template ::type> - CONSTCD14 string_literal(const char (&a)[N]) NOEXCEPT : p_{} { - for (std::size_t i = 0; i < N; ++i) p_[i] = a[i]; - } - - template ::value>::type> - CONSTCD14 string_literal(string_literal const& a) NOEXCEPT : p_{} { - for (std::size_t i = 0; i < N; ++i) p_[i] = a[i]; - } - - CONSTCD11 const CharT* data() const NOEXCEPT { return p_; } - CONSTCD11 std::size_t size() const NOEXCEPT { return N - 1; } - - CONSTCD11 const_iterator begin() const NOEXCEPT { return p_; } - CONSTCD11 const_iterator end() const NOEXCEPT { return p_ + N - 1; } - - CONSTCD11 CharT const& operator[](std::size_t n) const NOEXCEPT { return p_[n]; } - - template - friend std::basic_ostream& operator<<( - std::basic_ostream& os, const string_literal& s) { - return os << s.p_; - } - - template - friend CONSTCD14 string_literal< - typename std::conditional::type, - N1 + N2 - 1> - operator+(const string_literal& x, - const string_literal& y) NOEXCEPT; -}; - -template -CONSTCD11 inline string_literal operator+( - const string_literal& x, const string_literal& y) NOEXCEPT { - return string_literal(x[0], y[0]); -} - -template -CONSTCD11 inline string_literal operator+( - const string_literal& x, const string_literal& y) NOEXCEPT { - return string_literal(x[0], x[1], y[0]); -} - -template -CONSTCD14 inline string_literal< - typename std::conditional::type, - N1 + N2 - 1> -operator+(const string_literal& x, - const string_literal& y) NOEXCEPT { - using CT = - typename std::conditional::type; - - string_literal r; - std::size_t i = 0; - for (; i < N1 - 1; ++i) r.p_[i] = CT(x.p_[i]); - for (std::size_t j = 0; j < N2; ++j, ++i) r.p_[i] = CT(y.p_[j]); - - return r; -} - -template -inline std::basic_string operator+( - std::basic_string x, const string_literal& y) { - x.append(y.data(), y.size()); - return x; -} - -#if __cplusplus >= 201402 && (!defined(__EDG_VERSION__) || __EDG_VERSION__ > 411) && \ - (!defined(__SUNPRO_CC) || __SUNPRO_CC > 0x5150) - -template {} || std::is_same{} || - std::is_same{} || std::is_same{}>> -CONSTCD14 inline string_literal msl(CharT c) NOEXCEPT { - return string_literal{c}; -} - -CONSTCD14 -inline std::size_t to_string_len(std::intmax_t i) { - std::size_t r = 0; - do { - i /= 10; - ++r; - } while (i > 0); - return r; -} - -template - CONSTCD14 inline std::enable_if_t < - N<10, string_literal> msl() NOEXCEPT { - return msl(char(N % 10 + '0')); -} - -template -CONSTCD14 inline std::enable_if_t<10 <= N, string_literal> -msl() NOEXCEPT { - return msl() + msl(char(N % 10 + '0')); -} - -template -CONSTCD14 inline std::enable_if_t< - std::ratio::type::den != 1, - string_literal::type::num) + - to_string_len(std::ratio::type::den) + 4>> -msl(std::ratio) NOEXCEPT { - using R = typename std::ratio::type; - return msl(CharT{'['}) + msl() + msl(CharT{'/'}) + msl() + - msl(CharT{']'}); -} - -template -CONSTCD14 inline std::enable_if_t< - std::ratio::type::den == 1, - string_literal::type::num) + 3>> -msl(std::ratio) NOEXCEPT { - using R = typename std::ratio::type; - return msl(CharT{'['}) + msl() + msl(CharT{']'}); -} - -#else // __cplusplus < 201402 || (defined(__EDG_VERSION__) && __EDG_VERSION__ <= 411) - -inline std::string to_string(std::uint64_t x) { return std::to_string(x); } - -template -inline std::basic_string to_string(std::uint64_t x) { - auto y = std::to_string(x); - return std::basic_string(y.begin(), y.end()); -} - -template -inline typename std::enable_if::type::den != 1, - std::basic_string>::type -msl(std::ratio) { - using R = typename std::ratio::type; - return std::basic_string(1, '[') + to_string(R::num) + CharT{'/'} + - to_string(R::den) + CharT{']'}; -} - -template -inline typename std::enable_if::type::den == 1, - std::basic_string>::type -msl(std::ratio) { - using R = typename std::ratio::type; - return std::basic_string(1, '[') + to_string(R::num) + CharT{']'}; -} - -#endif // __cplusplus < 201402 || (defined(__EDG_VERSION__) && __EDG_VERSION__ <= 411) - -template -CONSTCD11 inline string_literal msl(std::atto) NOEXCEPT { - return string_literal{'a'}; -} - -template -CONSTCD11 inline string_literal msl(std::femto) NOEXCEPT { - return string_literal{'f'}; -} - -template -CONSTCD11 inline string_literal msl(std::pico) NOEXCEPT { - return string_literal{'p'}; -} - -template -CONSTCD11 inline string_literal msl(std::nano) NOEXCEPT { - return string_literal{'n'}; -} - -template -CONSTCD11 inline typename std::enable_if::value, - string_literal>::type -msl(std::micro) NOEXCEPT { - return string_literal{'\xC2', '\xB5'}; -} - -template -CONSTCD11 inline typename std::enable_if::value, - string_literal>::type -msl(std::micro) NOEXCEPT { - return string_literal{CharT{static_cast('\xB5')}}; -} - -template -CONSTCD11 inline string_literal msl(std::milli) NOEXCEPT { - return string_literal{'m'}; -} - -template -CONSTCD11 inline string_literal msl(std::centi) NOEXCEPT { - return string_literal{'c'}; -} - -template -CONSTCD11 inline string_literal msl(std::deca) NOEXCEPT { - return string_literal{'d', 'a'}; -} - -template -CONSTCD11 inline string_literal msl(std::deci) NOEXCEPT { - return string_literal{'d'}; -} - -template -CONSTCD11 inline string_literal msl(std::hecto) NOEXCEPT { - return string_literal{'h'}; -} - -template -CONSTCD11 inline string_literal msl(std::kilo) NOEXCEPT { - return string_literal{'k'}; -} - -template -CONSTCD11 inline string_literal msl(std::mega) NOEXCEPT { - return string_literal{'M'}; -} - -template -CONSTCD11 inline string_literal msl(std::giga) NOEXCEPT { - return string_literal{'G'}; -} - -template -CONSTCD11 inline string_literal msl(std::tera) NOEXCEPT { - return string_literal{'T'}; -} - -template -CONSTCD11 inline string_literal msl(std::peta) NOEXCEPT { - return string_literal{'P'}; -} - -template -CONSTCD11 inline string_literal msl(std::exa) NOEXCEPT { - return string_literal{'E'}; -} - -template -CONSTCD11 inline auto get_units(Period p) - -> decltype(msl(p) + string_literal{'s'}) { - return msl(p) + string_literal{'s'}; -} - -template -CONSTCD11 inline string_literal get_units(std::ratio<1>) { - return string_literal{'s'}; -} - -template -CONSTCD11 inline string_literal get_units(std::ratio<3600>) { - return string_literal{'h'}; -} - -template -CONSTCD11 inline string_literal get_units(std::ratio<60>) { - return string_literal{'m', 'i', 'n'}; -} - -template -CONSTCD11 inline string_literal get_units(std::ratio<86400>) { - return string_literal{'d'}; -} - -template > -struct make_string; - -template <> -struct make_string { - template - static std::string from(Rep n) { - return std::to_string(n); - } -}; - -template -struct make_string { - template - static std::basic_string from(Rep n) { - auto s = std::to_string(n); - return std::basic_string(s.begin(), s.end()); - } -}; - -template <> -struct make_string { - template - static std::wstring from(Rep n) { - return std::to_wstring(n); - } -}; - -template -struct make_string { - template - static std::basic_string from(Rep n) { - auto s = std::to_wstring(n); - return std::basic_string(s.begin(), s.end()); - } -}; - -} // namespace detail - -template -inline std::basic_ostream& operator<<( - std::basic_ostream& os, const std::chrono::duration& d) { - using namespace detail; - return os << make_string::from(d.count()) + - get_units(typename Period::type{}); -} - -} // namespace date - -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif - -#endif // DATE_H diff --git a/cpp/src/arrow/vendored/datetime/README.md b/cpp/src/arrow/vendored/datetime/README.md new file mode 100644 index 0000000000000..ff156ea310095 --- /dev/null +++ b/cpp/src/arrow/vendored/datetime/README.md @@ -0,0 +1,21 @@ + + +# Utilities for supporting date time functions + +Sources for datetime are adapted from Howard Hinnant's date library +(https://github.com/HowardHinnant/date). + +Sources are taken from v2.4.1 release of the above project. + diff --git a/cpp/src/arrow/vendored/datetime/date.h b/cpp/src/arrow/vendored/datetime/date.h new file mode 100644 index 0000000000000..f2889e416b054 --- /dev/null +++ b/cpp/src/arrow/vendored/datetime/date.h @@ -0,0 +1,8028 @@ +#ifndef DATE_H +#define DATE_H + +// The MIT License (MIT) +// +// Copyright (c) 2015, 2016, 2017 Howard Hinnant +// Copyright (c) 2016 Adrian Colomitchi +// Copyright (c) 2017 Florian Dang +// Copyright (c) 2017 Paul Thompson +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Our apologies. When the previous paragraph was written, lowercase had not yet +// been invented (that would involve another several millennia of evolution). +// We did not mean to shout. + +#ifndef HAS_STRING_VIEW +# if __cplusplus >= 201703 +# define HAS_STRING_VIEW 1 +# else +# define HAS_STRING_VIEW 0 +# endif +#endif // HAS_STRING_VIEW + +#include +#include +#include +#include +#include +#if !(__cplusplus >= 201402) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HAS_STRING_VIEW +# include +#endif +#include +#include + +#ifdef __GNUC__ +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wpedantic" +# if __GNUC__ < 5 + // GCC 4.9 Bug 61489 Wrong warning with -Wmissing-field-initializers +# pragma GCC diagnostic ignored "-Wmissing-field-initializers" +# endif +#endif + +namespace arrow +{ +namespace util +{ +namespace date +{ + +//---------------+ +// Configuration | +//---------------+ + +#ifndef ONLY_C_LOCALE +# define ONLY_C_LOCALE 0 +#endif + +#if defined(_MSC_VER) && (!defined(__clang__) || (_MSC_VER < 1910)) +// MSVC +# if _MSC_VER < 1910 +// before VS2017 +# define CONSTDATA const +# define CONSTCD11 +# define CONSTCD14 +# define NOEXCEPT _NOEXCEPT +# else +// VS2017 and later +# define CONSTDATA constexpr const +# define CONSTCD11 constexpr +# define CONSTCD14 constexpr +# define NOEXCEPT noexcept +# endif + +#elif defined(__SUNPRO_CC) && __SUNPRO_CC <= 0x5150 +// Oracle Developer Studio 12.6 and earlier +# define CONSTDATA constexpr const +# define CONSTCD11 constexpr +# define CONSTCD14 +# define NOEXCEPT noexcept + +#elif __cplusplus >= 201402 +// C++14 +# define CONSTDATA constexpr const +# define CONSTCD11 constexpr +# define CONSTCD14 constexpr +# define NOEXCEPT noexcept +#else +// C++11 +# define CONSTDATA constexpr const +# define CONSTCD11 constexpr +# define CONSTCD14 +# define NOEXCEPT noexcept +#endif + +#ifndef HAS_VOID_T +# if __cplusplus >= 201703 +# define HAS_VOID_T 1 +# else +# define HAS_VOID_T 0 +# endif +#endif // HAS_VOID_T + +// Protect from Oracle sun macro +#ifdef sun +# undef sun +#endif + +//-----------+ +// Interface | +//-----------+ + +// durations + +using days = std::chrono::duration + , std::chrono::hours::period>>; + +using weeks = std::chrono::duration + , days::period>>; + +using years = std::chrono::duration + , days::period>>; + +using months = std::chrono::duration + >>; + +// time_point + +template + using sys_time = std::chrono::time_point; + +using sys_days = sys_time; +using sys_seconds = sys_time; + +struct local_t {}; + +template + using local_time = std::chrono::time_point; + +using local_seconds = local_time; +using local_days = local_time; + +// types + +struct last_spec +{ + explicit last_spec() = default; +}; + +class day; +class month; +class year; + +class weekday; +class weekday_indexed; +class weekday_last; + +class month_day; +class month_day_last; +class month_weekday; +class month_weekday_last; + +class year_month; + +class year_month_day; +class year_month_day_last; +class year_month_weekday; +class year_month_weekday_last; + +// date composition operators + +CONSTCD11 year_month operator/(const year& y, const month& m) NOEXCEPT; +CONSTCD11 year_month operator/(const year& y, int m) NOEXCEPT; + +CONSTCD11 month_day operator/(const day& d, const month& m) NOEXCEPT; +CONSTCD11 month_day operator/(const day& d, int m) NOEXCEPT; +CONSTCD11 month_day operator/(const month& m, const day& d) NOEXCEPT; +CONSTCD11 month_day operator/(const month& m, int d) NOEXCEPT; +CONSTCD11 month_day operator/(int m, const day& d) NOEXCEPT; + +CONSTCD11 month_day_last operator/(const month& m, last_spec) NOEXCEPT; +CONSTCD11 month_day_last operator/(int m, last_spec) NOEXCEPT; +CONSTCD11 month_day_last operator/(last_spec, const month& m) NOEXCEPT; +CONSTCD11 month_day_last operator/(last_spec, int m) NOEXCEPT; + +CONSTCD11 month_weekday operator/(const month& m, const weekday_indexed& wdi) NOEXCEPT; +CONSTCD11 month_weekday operator/(int m, const weekday_indexed& wdi) NOEXCEPT; +CONSTCD11 month_weekday operator/(const weekday_indexed& wdi, const month& m) NOEXCEPT; +CONSTCD11 month_weekday operator/(const weekday_indexed& wdi, int m) NOEXCEPT; + +CONSTCD11 month_weekday_last operator/(const month& m, const weekday_last& wdl) NOEXCEPT; +CONSTCD11 month_weekday_last operator/(int m, const weekday_last& wdl) NOEXCEPT; +CONSTCD11 month_weekday_last operator/(const weekday_last& wdl, const month& m) NOEXCEPT; +CONSTCD11 month_weekday_last operator/(const weekday_last& wdl, int m) NOEXCEPT; + +CONSTCD11 year_month_day operator/(const year_month& ym, const day& d) NOEXCEPT; +CONSTCD11 year_month_day operator/(const year_month& ym, int d) NOEXCEPT; +CONSTCD11 year_month_day operator/(const year& y, const month_day& md) NOEXCEPT; +CONSTCD11 year_month_day operator/(int y, const month_day& md) NOEXCEPT; +CONSTCD11 year_month_day operator/(const month_day& md, const year& y) NOEXCEPT; +CONSTCD11 year_month_day operator/(const month_day& md, int y) NOEXCEPT; + +CONSTCD11 + year_month_day_last operator/(const year_month& ym, last_spec) NOEXCEPT; +CONSTCD11 + year_month_day_last operator/(const year& y, const month_day_last& mdl) NOEXCEPT; +CONSTCD11 + year_month_day_last operator/(int y, const month_day_last& mdl) NOEXCEPT; +CONSTCD11 + year_month_day_last operator/(const month_day_last& mdl, const year& y) NOEXCEPT; +CONSTCD11 + year_month_day_last operator/(const month_day_last& mdl, int y) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(const year_month& ym, const weekday_indexed& wdi) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(const year& y, const month_weekday& mwd) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(int y, const month_weekday& mwd) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(const month_weekday& mwd, const year& y) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator/(const month_weekday& mwd, int y) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(const year_month& ym, const weekday_last& wdl) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(const year& y, const month_weekday_last& mwdl) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(int y, const month_weekday_last& mwdl) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(const month_weekday_last& mwdl, const year& y) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator/(const month_weekday_last& mwdl, int y) NOEXCEPT; + +// Detailed interface + +// day + +class day +{ + unsigned char d_; + +public: + day() = default; + explicit CONSTCD11 day(unsigned d) NOEXCEPT; + + CONSTCD14 day& operator++() NOEXCEPT; + CONSTCD14 day operator++(int) NOEXCEPT; + CONSTCD14 day& operator--() NOEXCEPT; + CONSTCD14 day operator--(int) NOEXCEPT; + + CONSTCD14 day& operator+=(const days& d) NOEXCEPT; + CONSTCD14 day& operator-=(const days& d) NOEXCEPT; + + CONSTCD11 explicit operator unsigned() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator!=(const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator< (const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator> (const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator<=(const day& x, const day& y) NOEXCEPT; +CONSTCD11 bool operator>=(const day& x, const day& y) NOEXCEPT; + +CONSTCD11 day operator+(const day& x, const days& y) NOEXCEPT; +CONSTCD11 day operator+(const days& x, const day& y) NOEXCEPT; +CONSTCD11 day operator-(const day& x, const days& y) NOEXCEPT; +CONSTCD11 days operator-(const day& x, const day& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const day& d); + +// month + +class month +{ + unsigned char m_; + +public: + month() = default; + explicit CONSTCD11 month(unsigned m) NOEXCEPT; + + CONSTCD14 month& operator++() NOEXCEPT; + CONSTCD14 month operator++(int) NOEXCEPT; + CONSTCD14 month& operator--() NOEXCEPT; + CONSTCD14 month operator--(int) NOEXCEPT; + + CONSTCD14 month& operator+=(const months& m) NOEXCEPT; + CONSTCD14 month& operator-=(const months& m) NOEXCEPT; + + CONSTCD11 explicit operator unsigned() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator!=(const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator< (const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator> (const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator<=(const month& x, const month& y) NOEXCEPT; +CONSTCD11 bool operator>=(const month& x, const month& y) NOEXCEPT; + +CONSTCD14 month operator+(const month& x, const months& y) NOEXCEPT; +CONSTCD14 month operator+(const months& x, const month& y) NOEXCEPT; +CONSTCD14 month operator-(const month& x, const months& y) NOEXCEPT; +CONSTCD14 months operator-(const month& x, const month& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month& m); + +// year + +class year +{ + short y_; + +public: + year() = default; + explicit CONSTCD11 year(int y) NOEXCEPT; + + CONSTCD14 year& operator++() NOEXCEPT; + CONSTCD14 year operator++(int) NOEXCEPT; + CONSTCD14 year& operator--() NOEXCEPT; + CONSTCD14 year operator--(int) NOEXCEPT; + + CONSTCD14 year& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 year operator-() const NOEXCEPT; + CONSTCD11 year operator+() const NOEXCEPT; + + CONSTCD11 bool is_leap() const NOEXCEPT; + + CONSTCD11 explicit operator int() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; + + static CONSTCD11 year min() NOEXCEPT; + static CONSTCD11 year max() NOEXCEPT; +}; + +CONSTCD11 bool operator==(const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator!=(const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator< (const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator> (const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator<=(const year& x, const year& y) NOEXCEPT; +CONSTCD11 bool operator>=(const year& x, const year& y) NOEXCEPT; + +CONSTCD11 year operator+(const year& x, const years& y) NOEXCEPT; +CONSTCD11 year operator+(const years& x, const year& y) NOEXCEPT; +CONSTCD11 year operator-(const year& x, const years& y) NOEXCEPT; +CONSTCD11 years operator-(const year& x, const year& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year& y); + +// weekday + +class weekday +{ + unsigned char wd_; +public: + weekday() = default; + explicit CONSTCD11 weekday(unsigned wd) NOEXCEPT; + CONSTCD11 weekday(const sys_days& dp) NOEXCEPT; + CONSTCD11 explicit weekday(const local_days& dp) NOEXCEPT; + + CONSTCD14 weekday& operator++() NOEXCEPT; + CONSTCD14 weekday operator++(int) NOEXCEPT; + CONSTCD14 weekday& operator--() NOEXCEPT; + CONSTCD14 weekday operator--(int) NOEXCEPT; + + CONSTCD14 weekday& operator+=(const days& d) NOEXCEPT; + CONSTCD14 weekday& operator-=(const days& d) NOEXCEPT; + + CONSTCD11 explicit operator unsigned() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; + + CONSTCD11 weekday_indexed operator[](unsigned index) const NOEXCEPT; + CONSTCD11 weekday_last operator[](last_spec) const NOEXCEPT; + +private: + static CONSTCD11 unsigned char weekday_from_days(int z) NOEXCEPT; +}; + +CONSTCD11 bool operator==(const weekday& x, const weekday& y) NOEXCEPT; +CONSTCD11 bool operator!=(const weekday& x, const weekday& y) NOEXCEPT; + +CONSTCD14 weekday operator+(const weekday& x, const days& y) NOEXCEPT; +CONSTCD14 weekday operator+(const days& x, const weekday& y) NOEXCEPT; +CONSTCD14 weekday operator-(const weekday& x, const days& y) NOEXCEPT; +CONSTCD14 days operator-(const weekday& x, const weekday& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday& wd); + +// weekday_indexed + +class weekday_indexed +{ + unsigned char wd_ : 4; + unsigned char index_ : 4; + +public: + weekday_indexed() = default; + CONSTCD11 weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT; + + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 unsigned index() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT; +CONSTCD11 bool operator!=(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday_indexed& wdi); + +// weekday_last + +class weekday_last +{ + date::weekday wd_; + +public: + explicit CONSTCD11 weekday_last(const date::weekday& wd) NOEXCEPT; + + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const weekday_last& x, const weekday_last& y) NOEXCEPT; +CONSTCD11 bool operator!=(const weekday_last& x, const weekday_last& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday_last& wdl); + +// year_month + +class year_month +{ + date::year y_; + date::month m_; + +public: + year_month() = default; + CONSTCD11 year_month(const date::year& y, const date::month& m) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + + CONSTCD14 year_month& operator+=(const months& dm) NOEXCEPT; + CONSTCD14 year_month& operator-=(const months& dm) NOEXCEPT; + CONSTCD14 year_month& operator+=(const years& dy) NOEXCEPT; + CONSTCD14 year_month& operator-=(const years& dy) NOEXCEPT; + + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator!=(const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator< (const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator> (const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator<=(const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 bool operator>=(const year_month& x, const year_month& y) NOEXCEPT; + +CONSTCD14 year_month operator+(const year_month& ym, const months& dm) NOEXCEPT; +CONSTCD14 year_month operator+(const months& dm, const year_month& ym) NOEXCEPT; +CONSTCD14 year_month operator-(const year_month& ym, const months& dm) NOEXCEPT; + +CONSTCD11 months operator-(const year_month& x, const year_month& y) NOEXCEPT; +CONSTCD11 year_month operator+(const year_month& ym, const years& dy) NOEXCEPT; +CONSTCD11 year_month operator+(const years& dy, const year_month& ym) NOEXCEPT; +CONSTCD11 year_month operator-(const year_month& ym, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month& ym); + +// month_day + +class month_day +{ + date::month m_; + date::day d_; + +public: + month_day() = default; + CONSTCD11 month_day(const date::month& m, const date::day& d) NOEXCEPT; + + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::day day() const NOEXCEPT; + + CONSTCD14 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator!=(const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator< (const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator> (const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator<=(const month_day& x, const month_day& y) NOEXCEPT; +CONSTCD11 bool operator>=(const month_day& x, const month_day& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_day& md); + +// month_day_last + +class month_day_last +{ + date::month m_; + +public: + CONSTCD11 explicit month_day_last(const date::month& m) NOEXCEPT; + + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator!=(const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator< (const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator> (const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator<=(const month_day_last& x, const month_day_last& y) NOEXCEPT; +CONSTCD11 bool operator>=(const month_day_last& x, const month_day_last& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_day_last& mdl); + +// month_weekday + +class month_weekday +{ + date::month m_; + date::weekday_indexed wdi_; +public: + CONSTCD11 month_weekday(const date::month& m, + const date::weekday_indexed& wdi) NOEXCEPT; + + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; + + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const month_weekday& x, const month_weekday& y) NOEXCEPT; +CONSTCD11 bool operator!=(const month_weekday& x, const month_weekday& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_weekday& mwd); + +// month_weekday_last + +class month_weekday_last +{ + date::month m_; + date::weekday_last wdl_; + +public: + CONSTCD11 month_weekday_last(const date::month& m, + const date::weekday_last& wd) NOEXCEPT; + + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; + + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 + bool operator==(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; +CONSTCD11 + bool operator!=(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_weekday_last& mwdl); + +// class year_month_day + +class year_month_day +{ + date::year y_; + date::month m_; + date::day d_; + +public: + year_month_day() = default; + CONSTCD11 year_month_day(const date::year& y, const date::month& m, + const date::day& d) NOEXCEPT; + CONSTCD14 year_month_day(const year_month_day_last& ymdl) NOEXCEPT; + + CONSTCD14 year_month_day(sys_days dp) NOEXCEPT; + CONSTCD14 explicit year_month_day(local_days dp) NOEXCEPT; + + CONSTCD14 year_month_day& operator+=(const months& m) NOEXCEPT; + CONSTCD14 year_month_day& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_day& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_day& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::day day() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD14 bool ok() const NOEXCEPT; + +private: + static CONSTCD14 year_month_day from_days(days dp) NOEXCEPT; + CONSTCD14 days to_days() const NOEXCEPT; +}; + +CONSTCD11 bool operator==(const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator!=(const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator< (const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator> (const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator<=(const year_month_day& x, const year_month_day& y) NOEXCEPT; +CONSTCD11 bool operator>=(const year_month_day& x, const year_month_day& y) NOEXCEPT; + +CONSTCD14 year_month_day operator+(const year_month_day& ymd, const months& dm) NOEXCEPT; +CONSTCD14 year_month_day operator+(const months& dm, const year_month_day& ymd) NOEXCEPT; +CONSTCD14 year_month_day operator-(const year_month_day& ymd, const months& dm) NOEXCEPT; +CONSTCD11 year_month_day operator+(const year_month_day& ymd, const years& dy) NOEXCEPT; +CONSTCD11 year_month_day operator+(const years& dy, const year_month_day& ymd) NOEXCEPT; +CONSTCD11 year_month_day operator-(const year_month_day& ymd, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_day& ymd); + +// year_month_day_last + +class year_month_day_last +{ + date::year y_; + date::month_day_last mdl_; + +public: + CONSTCD11 year_month_day_last(const date::year& y, + const date::month_day_last& mdl) NOEXCEPT; + + CONSTCD14 year_month_day_last& operator+=(const months& m) NOEXCEPT; + CONSTCD14 year_month_day_last& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_day_last& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_day_last& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::month_day_last month_day_last() const NOEXCEPT; + CONSTCD14 date::day day() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; +}; + +CONSTCD11 + bool operator==(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 + bool operator!=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 + bool operator< (const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 + bool operator> (const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 + bool operator<=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; +CONSTCD11 + bool operator>=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT; + +CONSTCD14 +year_month_day_last +operator+(const year_month_day_last& ymdl, const months& dm) NOEXCEPT; + +CONSTCD14 +year_month_day_last +operator+(const months& dm, const year_month_day_last& ymdl) NOEXCEPT; + +CONSTCD11 +year_month_day_last +operator+(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; + +CONSTCD11 +year_month_day_last +operator+(const years& dy, const year_month_day_last& ymdl) NOEXCEPT; + +CONSTCD14 +year_month_day_last +operator-(const year_month_day_last& ymdl, const months& dm) NOEXCEPT; + +CONSTCD11 +year_month_day_last +operator-(const year_month_day_last& ymdl, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_day_last& ymdl); + +// year_month_weekday + +class year_month_weekday +{ + date::year y_; + date::month m_; + date::weekday_indexed wdi_; + +public: + year_month_weekday() = default; + CONSTCD11 year_month_weekday(const date::year& y, const date::month& m, + const date::weekday_indexed& wdi) NOEXCEPT; + CONSTCD14 year_month_weekday(const sys_days& dp) NOEXCEPT; + CONSTCD14 explicit year_month_weekday(const local_days& dp) NOEXCEPT; + + CONSTCD14 year_month_weekday& operator+=(const months& m) NOEXCEPT; + CONSTCD14 year_month_weekday& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_weekday& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_weekday& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 unsigned index() const NOEXCEPT; + CONSTCD11 date::weekday_indexed weekday_indexed() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD14 bool ok() const NOEXCEPT; + +private: + static CONSTCD14 year_month_weekday from_days(days dp) NOEXCEPT; + CONSTCD14 days to_days() const NOEXCEPT; +}; + +CONSTCD11 + bool operator==(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; +CONSTCD11 + bool operator!=(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT; + +CONSTCD14 +year_month_weekday +operator+(const year_month_weekday& ymwd, const months& dm) NOEXCEPT; + +CONSTCD14 +year_month_weekday +operator+(const months& dm, const year_month_weekday& ymwd) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator+(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator+(const years& dy, const year_month_weekday& ymwd) NOEXCEPT; + +CONSTCD14 +year_month_weekday +operator-(const year_month_weekday& ymwd, const months& dm) NOEXCEPT; + +CONSTCD11 +year_month_weekday +operator-(const year_month_weekday& ymwd, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_weekday& ymwdi); + +// year_month_weekday_last + +class year_month_weekday_last +{ + date::year y_; + date::month m_; + date::weekday_last wdl_; + +public: + CONSTCD11 year_month_weekday_last(const date::year& y, const date::month& m, + const date::weekday_last& wdl) NOEXCEPT; + + CONSTCD14 year_month_weekday_last& operator+=(const months& m) NOEXCEPT; + CONSTCD14 year_month_weekday_last& operator-=(const months& m) NOEXCEPT; + CONSTCD14 year_month_weekday_last& operator+=(const years& y) NOEXCEPT; + CONSTCD14 year_month_weekday_last& operator-=(const years& y) NOEXCEPT; + + CONSTCD11 date::year year() const NOEXCEPT; + CONSTCD11 date::month month() const NOEXCEPT; + CONSTCD11 date::weekday weekday() const NOEXCEPT; + CONSTCD11 date::weekday_last weekday_last() const NOEXCEPT; + + CONSTCD14 operator sys_days() const NOEXCEPT; + CONSTCD14 explicit operator local_days() const NOEXCEPT; + CONSTCD11 bool ok() const NOEXCEPT; + +private: + CONSTCD14 days to_days() const NOEXCEPT; +}; + +CONSTCD11 +bool +operator==(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT; + +CONSTCD11 +bool +operator!=(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT; + +CONSTCD14 +year_month_weekday_last +operator+(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT; + +CONSTCD14 +year_month_weekday_last +operator+(const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator+(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator+(const years& dy, const year_month_weekday_last& ymwdl) NOEXCEPT; + +CONSTCD14 +year_month_weekday_last +operator-(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT; + +CONSTCD11 +year_month_weekday_last +operator-(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_weekday_last& ymwdl); + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +inline namespace literals +{ + +CONSTCD11 date::day operator "" _d(unsigned long long d) NOEXCEPT; +CONSTCD11 date::year operator "" _y(unsigned long long y) NOEXCEPT; + +// CONSTDATA date::month jan{1}; +// CONSTDATA date::month feb{2}; +// CONSTDATA date::month mar{3}; +// CONSTDATA date::month apr{4}; +// CONSTDATA date::month may{5}; +// CONSTDATA date::month jun{6}; +// CONSTDATA date::month jul{7}; +// CONSTDATA date::month aug{8}; +// CONSTDATA date::month sep{9}; +// CONSTDATA date::month oct{10}; +// CONSTDATA date::month nov{11}; +// CONSTDATA date::month dec{12}; +// +// CONSTDATA date::weekday sun{0u}; +// CONSTDATA date::weekday mon{1u}; +// CONSTDATA date::weekday tue{2u}; +// CONSTDATA date::weekday wed{3u}; +// CONSTDATA date::weekday thu{4u}; +// CONSTDATA date::weekday fri{5u}; +// CONSTDATA date::weekday sat{6u}; + +} // inline namespace literals +#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) + +#if HAS_VOID_T + +template > +struct is_clock + : std::false_type +{}; + +template +struct is_clock> + : std::true_type +{}; + +#endif // HAS_VOID_T + +//----------------+ +// Implementation | +//----------------+ + +// utilities +namespace detail { + +template> +class save_stream +{ + std::basic_ostream& os_; + CharT fill_; + std::ios::fmtflags flags_; + std::locale loc_; + +public: + ~save_stream() + { + os_.fill(fill_); + os_.flags(flags_); + os_.imbue(loc_); + } + + save_stream(const save_stream&) = delete; + save_stream& operator=(const save_stream&) = delete; + + explicit save_stream(std::basic_ostream& os) + : os_(os) + , fill_(os.fill()) + , flags_(os.flags()) + , loc_(os.getloc()) + {} +}; + +template +struct choose_trunc_type +{ + static const int digits = std::numeric_limits::digits; + using type = typename std::conditional + < + digits < 32, + std::int32_t, + typename std::conditional + < + digits < 64, + std::int64_t, +#ifdef __SIZEOF_INT128__ + __int128 +#else + std::int64_t +#endif + >::type + >::type; +}; + +template +CONSTCD11 +inline +typename std::enable_if +< + !std::chrono::treat_as_floating_point::value, + T +>::type +trunc(T t) NOEXCEPT +{ + return t; +} + +template +CONSTCD14 +inline +typename std::enable_if +< + std::chrono::treat_as_floating_point::value, + T +>::type +trunc(T t) NOEXCEPT +{ + using namespace std; + using I = typename choose_trunc_type::type; + CONSTDATA auto digits = numeric_limits::digits; + static_assert(digits < numeric_limits::digits, ""); + CONSTDATA auto max = I{1} << (digits-1); + CONSTDATA auto min = -max; + const auto negative = t < T{0}; + if (min <= t && t <= max && t != 0 && t == t) + { + t = static_cast(static_cast(t)); + if (t == 0 && negative) + t = -t; + } + return t; +} + +template +struct static_gcd +{ + static const std::intmax_t value = static_gcd::value; +}; + +template +struct static_gcd +{ + static const std::intmax_t value = Xp; +}; + +template <> +struct static_gcd<0, 0> +{ + static const std::intmax_t value = 1; +}; + +template +struct no_overflow +{ +private: + static const std::intmax_t gcd_n1_n2 = static_gcd::value; + static const std::intmax_t gcd_d1_d2 = static_gcd::value; + static const std::intmax_t n1 = R1::num / gcd_n1_n2; + static const std::intmax_t d1 = R1::den / gcd_d1_d2; + static const std::intmax_t n2 = R2::num / gcd_n1_n2; + static const std::intmax_t d2 = R2::den / gcd_d1_d2; + static const std::intmax_t max = -((std::intmax_t(1) << + (sizeof(std::intmax_t) * CHAR_BIT - 1)) + 1); + + template + struct mul // overflow == false + { + static const std::intmax_t value = Xp * Yp; + }; + + template + struct mul + { + static const std::intmax_t value = 1; + }; + +public: + static const bool value = (n1 <= max / d2) && (n2 <= max / d1); + typedef std::ratio::value, + mul::value> type; +}; + +} // detail + +// trunc towards zero +template +CONSTCD11 +inline +typename std::enable_if +< + detail::no_overflow::value, + To +>::type +trunc(const std::chrono::duration& d) +{ + return To{detail::trunc(std::chrono::duration_cast(d).count())}; +} + +template +CONSTCD11 +inline +typename std::enable_if +< + !detail::no_overflow::value, + To +>::type +trunc(const std::chrono::duration& d) +{ + using namespace std::chrono; + using rep = typename std::common_type::type; + return To{detail::trunc(duration_cast(duration_cast>(d)).count())}; +} + +#ifndef HAS_CHRONO_ROUNDING +# if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190023918 || (_MSC_FULL_VER >= 190000000 && defined (__clang__))) +# define HAS_CHRONO_ROUNDING 1 +# elif defined(__cpp_lib_chrono) && __cplusplus > 201402 && __cpp_lib_chrono >= 201510 +# define HAS_CHRONO_ROUNDING 1 +# elif defined(_LIBCPP_VERSION) && __cplusplus > 201402 && _LIBCPP_VERSION >= 3800 +# define HAS_CHRONO_ROUNDING 1 +# else +# define HAS_CHRONO_ROUNDING 0 +# endif +#endif // HAS_CHRONO_ROUNDING + +#if HAS_CHRONO_ROUNDING == 0 + +// round down +template +CONSTCD14 +inline +typename std::enable_if +< + detail::no_overflow::value, + To +>::type +floor(const std::chrono::duration& d) +{ + auto t = trunc(d); + if (t > d) + return t - To{1}; + return t; +} + +template +CONSTCD14 +inline +typename std::enable_if +< + !detail::no_overflow::value, + To +>::type +floor(const std::chrono::duration& d) +{ + using namespace std::chrono; + using rep = typename std::common_type::type; + return floor(floor>(d)); +} + +// round to nearest, to even on tie +template +CONSTCD14 +inline +To +round(const std::chrono::duration& d) +{ + auto t0 = floor(d); + auto t1 = t0 + To{1}; + if (t1 == To{0} && t0 < To{0}) + t1 = -t1; + auto diff0 = d - t0; + auto diff1 = t1 - d; + if (diff0 == diff1) + { + if (t0 - trunc(t0/2)*2 == To{0}) + return t0; + return t1; + } + if (diff0 < diff1) + return t0; + return t1; +} + +// round up +template +CONSTCD14 +inline +To +ceil(const std::chrono::duration& d) +{ + auto t = trunc(d); + if (t < d) + return t + To{1}; + return t; +} + +template ::is_signed + >::type> +CONSTCD11 +std::chrono::duration +abs(std::chrono::duration d) +{ + return d >= d.zero() ? d : -d; +} + +// round down +template +CONSTCD11 +inline +std::chrono::time_point +floor(const std::chrono::time_point& tp) +{ + using std::chrono::time_point; + return time_point{date::floor(tp.time_since_epoch())}; +} + +// round to nearest, to even on tie +template +CONSTCD11 +inline +std::chrono::time_point +round(const std::chrono::time_point& tp) +{ + using std::chrono::time_point; + return time_point{round(tp.time_since_epoch())}; +} + +// round up +template +CONSTCD11 +inline +std::chrono::time_point +ceil(const std::chrono::time_point& tp) +{ + using std::chrono::time_point; + return time_point{ceil(tp.time_since_epoch())}; +} + +#else // HAS_CHRONO_ROUNDING == 1 + +using std::chrono::floor; +using std::chrono::ceil; +using std::chrono::round; +using std::chrono::abs; + +#endif // HAS_CHRONO_ROUNDING + +// trunc towards zero +template +CONSTCD11 +inline +std::chrono::time_point +trunc(const std::chrono::time_point& tp) +{ + using std::chrono::time_point; + return time_point{trunc(tp.time_since_epoch())}; +} + +// day + +CONSTCD11 inline day::day(unsigned d) NOEXCEPT : d_(static_cast(d)) {} +CONSTCD14 inline day& day::operator++() NOEXCEPT {++d_; return *this;} +CONSTCD14 inline day day::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} +CONSTCD14 inline day& day::operator--() NOEXCEPT {--d_; return *this;} +CONSTCD14 inline day day::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} +CONSTCD14 inline day& day::operator+=(const days& d) NOEXCEPT {*this = *this + d; return *this;} +CONSTCD14 inline day& day::operator-=(const days& d) NOEXCEPT {*this = *this - d; return *this;} +CONSTCD11 inline day::operator unsigned() const NOEXCEPT {return d_;} +CONSTCD11 inline bool day::ok() const NOEXCEPT {return 1 <= d_ && d_ <= 31;} + +CONSTCD11 +inline +bool +operator==(const day& x, const day& y) NOEXCEPT +{ + return static_cast(x) == static_cast(y); +} + +CONSTCD11 +inline +bool +operator!=(const day& x, const day& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const day& x, const day& y) NOEXCEPT +{ + return static_cast(x) < static_cast(y); +} + +CONSTCD11 +inline +bool +operator>(const day& x, const day& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const day& x, const day& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const day& x, const day& y) NOEXCEPT +{ + return !(x < y); +} + +CONSTCD11 +inline +days +operator-(const day& x, const day& y) NOEXCEPT +{ + return days{static_cast(static_cast(x) + - static_cast(y))}; +} + +CONSTCD11 +inline +day +operator+(const day& x, const days& y) NOEXCEPT +{ + return day{static_cast(x) + static_cast(y.count())}; +} + +CONSTCD11 +inline +day +operator+(const days& x, const day& y) NOEXCEPT +{ + return y + x; +} + +CONSTCD11 +inline +day +operator-(const day& x, const days& y) NOEXCEPT +{ + return x + -y; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const day& d) +{ + detail::save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << static_cast(d); + if (!d.ok()) + os << " is not a valid day"; + return os; +} + +// month + +CONSTCD11 inline month::month(unsigned m) NOEXCEPT : m_(static_cast(m)) {} +CONSTCD14 inline month& month::operator++() NOEXCEPT {*this += months{1}; return *this;} +CONSTCD14 inline month month::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} +CONSTCD14 inline month& month::operator--() NOEXCEPT {*this -= months{1}; return *this;} +CONSTCD14 inline month month::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} + +CONSTCD14 +inline +month& +month::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +CONSTCD14 +inline +month& +month::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD11 inline month::operator unsigned() const NOEXCEPT {return m_;} +CONSTCD11 inline bool month::ok() const NOEXCEPT {return 1 <= m_ && m_ <= 12;} + +CONSTCD11 +inline +bool +operator==(const month& x, const month& y) NOEXCEPT +{ + return static_cast(x) == static_cast(y); +} + +CONSTCD11 +inline +bool +operator!=(const month& x, const month& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const month& x, const month& y) NOEXCEPT +{ + return static_cast(x) < static_cast(y); +} + +CONSTCD11 +inline +bool +operator>(const month& x, const month& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const month& x, const month& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const month& x, const month& y) NOEXCEPT +{ + return !(x < y); +} + +CONSTCD14 +inline +months +operator-(const month& x, const month& y) NOEXCEPT +{ + auto const d = static_cast(x) - static_cast(y); + return months(d <= 11 ? d : d + 12); +} + +CONSTCD14 +inline +month +operator+(const month& x, const months& y) NOEXCEPT +{ + auto const mu = static_cast(static_cast(x)) + (y.count() - 1); + auto const yr = (mu >= 0 ? mu : mu-11) / 12; + return month{static_cast(mu - yr * 12 + 1)}; +} + +CONSTCD14 +inline +month +operator+(const months& x, const month& y) NOEXCEPT +{ + return y + x; +} + +CONSTCD14 +inline +month +operator-(const month& x, const months& y) NOEXCEPT +{ + return x + -y; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month& m) +{ + if (m.ok()) + { + CharT fmt[] = {'%', 'b', 0}; + os << format(os.getloc(), fmt, m); + } + else + os << static_cast(m) << " is not a valid month"; + return os; +} + +// year + +CONSTCD11 inline year::year(int y) NOEXCEPT : y_(static_cast(y)) {} +CONSTCD14 inline year& year::operator++() NOEXCEPT {++y_; return *this;} +CONSTCD14 inline year year::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} +CONSTCD14 inline year& year::operator--() NOEXCEPT {--y_; return *this;} +CONSTCD14 inline year year::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} +CONSTCD14 inline year& year::operator+=(const years& y) NOEXCEPT {*this = *this + y; return *this;} +CONSTCD14 inline year& year::operator-=(const years& y) NOEXCEPT {*this = *this - y; return *this;} +CONSTCD11 inline year year::operator-() const NOEXCEPT {return year{-y_};} +CONSTCD11 inline year year::operator+() const NOEXCEPT {return *this;} + +CONSTCD11 +inline +bool +year::is_leap() const NOEXCEPT +{ + return y_ % 4 == 0 && (y_ % 100 != 0 || y_ % 400 == 0); +} + +CONSTCD11 inline year::operator int() const NOEXCEPT {return y_;} + +CONSTCD11 +inline +bool +year::ok() const NOEXCEPT +{ + return y_ != std::numeric_limits::min(); +} + +CONSTCD11 +inline +year +year::min() NOEXCEPT +{ + return year{-32767}; +} + +CONSTCD11 +inline +year +year::max() NOEXCEPT +{ + return year{32767}; +} + +CONSTCD11 +inline +bool +operator==(const year& x, const year& y) NOEXCEPT +{ + return static_cast(x) == static_cast(y); +} + +CONSTCD11 +inline +bool +operator!=(const year& x, const year& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const year& x, const year& y) NOEXCEPT +{ + return static_cast(x) < static_cast(y); +} + +CONSTCD11 +inline +bool +operator>(const year& x, const year& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const year& x, const year& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const year& x, const year& y) NOEXCEPT +{ + return !(x < y); +} + +CONSTCD11 +inline +years +operator-(const year& x, const year& y) NOEXCEPT +{ + return years{static_cast(x) - static_cast(y)}; +} + +CONSTCD11 +inline +year +operator+(const year& x, const years& y) NOEXCEPT +{ + return year{static_cast(x) + y.count()}; +} + +CONSTCD11 +inline +year +operator+(const years& x, const year& y) NOEXCEPT +{ + return y + x; +} + +CONSTCD11 +inline +year +operator-(const year& x, const years& y) NOEXCEPT +{ + return year{static_cast(x) - y.count()}; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year& y) +{ + detail::save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::internal); + os.width(4 + (y < year{0})); + os << static_cast(y); + if (!y.ok()) + os << " is not a valid year"; + return os; +} + +// weekday + +CONSTCD11 +inline +unsigned char +weekday::weekday_from_days(int z) NOEXCEPT +{ + return static_cast(static_cast( + z >= -4 ? (z+4) % 7 : (z+5) % 7 + 6)); +} + +CONSTCD11 +inline +weekday::weekday(unsigned wd) NOEXCEPT + : wd_(static_cast(wd)) + {} + +CONSTCD11 +inline +weekday::weekday(const sys_days& dp) NOEXCEPT + : wd_(weekday_from_days(dp.time_since_epoch().count())) + {} + +CONSTCD11 +inline +weekday::weekday(const local_days& dp) NOEXCEPT + : wd_(weekday_from_days(dp.time_since_epoch().count())) + {} + +CONSTCD14 inline weekday& weekday::operator++() NOEXCEPT {*this += days{1}; return *this;} +CONSTCD14 inline weekday weekday::operator++(int) NOEXCEPT {auto tmp(*this); ++(*this); return tmp;} +CONSTCD14 inline weekday& weekday::operator--() NOEXCEPT {*this -= days{1}; return *this;} +CONSTCD14 inline weekday weekday::operator--(int) NOEXCEPT {auto tmp(*this); --(*this); return tmp;} + +CONSTCD14 +inline +weekday& +weekday::operator+=(const days& d) NOEXCEPT +{ + *this = *this + d; + return *this; +} + +CONSTCD14 +inline +weekday& +weekday::operator-=(const days& d) NOEXCEPT +{ + *this = *this - d; + return *this; +} + +CONSTCD11 +inline +weekday::operator unsigned() const NOEXCEPT +{ + return static_cast(wd_); +} + +CONSTCD11 inline bool weekday::ok() const NOEXCEPT {return wd_ <= 6;} + +CONSTCD11 +inline +bool +operator==(const weekday& x, const weekday& y) NOEXCEPT +{ + return static_cast(x) == static_cast(y); +} + +CONSTCD11 +inline +bool +operator!=(const weekday& x, const weekday& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD14 +inline +days +operator-(const weekday& x, const weekday& y) NOEXCEPT +{ + auto const diff = static_cast(x) - static_cast(y); + return days{diff <= 6 ? diff : diff + 7}; +} + +CONSTCD14 +inline +weekday +operator+(const weekday& x, const days& y) NOEXCEPT +{ + auto const wdu = static_cast(static_cast(x)) + y.count(); + auto const wk = (wdu >= 0 ? wdu : wdu-6) / 7; + return weekday{static_cast(wdu - wk * 7)}; +} + +CONSTCD14 +inline +weekday +operator+(const days& x, const weekday& y) NOEXCEPT +{ + return y + x; +} + +CONSTCD14 +inline +weekday +operator-(const weekday& x, const days& y) NOEXCEPT +{ + return x + -y; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday& wd) +{ + if (wd.ok()) + { + CharT fmt[] = {'%', 'a', 0}; + os << format(fmt, wd); + } + else + os << static_cast(wd) << " is not a valid weekday"; + return os; +} + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +inline namespace literals +{ + +CONSTCD11 +inline +date::day +operator "" _d(unsigned long long d) NOEXCEPT +{ + return date::day{static_cast(d)}; +} + +CONSTCD11 +inline +date::year +operator "" _y(unsigned long long y) NOEXCEPT +{ + return date::year(static_cast(y)); +} +#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) + +CONSTDATA date::last_spec last{}; + +CONSTDATA date::month jan{1}; +CONSTDATA date::month feb{2}; +CONSTDATA date::month mar{3}; +CONSTDATA date::month apr{4}; +CONSTDATA date::month may{5}; +CONSTDATA date::month jun{6}; +CONSTDATA date::month jul{7}; +CONSTDATA date::month aug{8}; +CONSTDATA date::month sep{9}; +CONSTDATA date::month oct{10}; +CONSTDATA date::month nov{11}; +CONSTDATA date::month dec{12}; + +CONSTDATA date::weekday sun{0u}; +CONSTDATA date::weekday mon{1u}; +CONSTDATA date::weekday tue{2u}; +CONSTDATA date::weekday wed{3u}; +CONSTDATA date::weekday thu{4u}; +CONSTDATA date::weekday fri{5u}; +CONSTDATA date::weekday sat{6u}; + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +} // inline namespace literals +#endif + +CONSTDATA date::month January{1}; +CONSTDATA date::month February{2}; +CONSTDATA date::month March{3}; +CONSTDATA date::month April{4}; +CONSTDATA date::month May{5}; +CONSTDATA date::month June{6}; +CONSTDATA date::month July{7}; +CONSTDATA date::month August{8}; +CONSTDATA date::month September{9}; +CONSTDATA date::month October{10}; +CONSTDATA date::month November{11}; +CONSTDATA date::month December{12}; + +CONSTDATA date::weekday Sunday{0u}; +CONSTDATA date::weekday Monday{1u}; +CONSTDATA date::weekday Tuesday{2u}; +CONSTDATA date::weekday Wednesday{3u}; +CONSTDATA date::weekday Thursday{4u}; +CONSTDATA date::weekday Friday{5u}; +CONSTDATA date::weekday Saturday{6u}; + +// weekday_indexed + +CONSTCD11 +inline +weekday +weekday_indexed::weekday() const NOEXCEPT +{ + return date::weekday{static_cast(wd_)}; +} + +CONSTCD11 inline unsigned weekday_indexed::index() const NOEXCEPT {return index_;} + +CONSTCD11 +inline +bool +weekday_indexed::ok() const NOEXCEPT +{ + return weekday().ok() && 1 <= index_ && index_ <= 5; +} + +#ifdef __GNUC__ +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wconversion" +#endif // __GNUC__ + +CONSTCD11 +inline +weekday_indexed::weekday_indexed(const date::weekday& wd, unsigned index) NOEXCEPT + : wd_(static_cast(static_cast(wd))) + , index_(static_cast(index)) + {} + +#ifdef __GNUC__ +# pragma GCC diagnostic pop +#endif // __GNUC__ + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday_indexed& wdi) +{ + os << wdi.weekday() << '[' << wdi.index(); + if (!(1 <= wdi.index() && wdi.index() <= 5)) + os << " is not a valid index"; + os << ']'; + return os; +} + +CONSTCD11 +inline +weekday_indexed +weekday::operator[](unsigned index) const NOEXCEPT +{ + return {*this, index}; +} + +CONSTCD11 +inline +bool +operator==(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT +{ + return x.weekday() == y.weekday() && x.index() == y.index(); +} + +CONSTCD11 +inline +bool +operator!=(const weekday_indexed& x, const weekday_indexed& y) NOEXCEPT +{ + return !(x == y); +} + +// weekday_last + +CONSTCD11 inline date::weekday weekday_last::weekday() const NOEXCEPT {return wd_;} +CONSTCD11 inline bool weekday_last::ok() const NOEXCEPT {return wd_.ok();} +CONSTCD11 inline weekday_last::weekday_last(const date::weekday& wd) NOEXCEPT : wd_(wd) {} + +CONSTCD11 +inline +bool +operator==(const weekday_last& x, const weekday_last& y) NOEXCEPT +{ + return x.weekday() == y.weekday(); +} + +CONSTCD11 +inline +bool +operator!=(const weekday_last& x, const weekday_last& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const weekday_last& wdl) +{ + return os << wdl.weekday() << "[last]"; +} + +CONSTCD11 +inline +weekday_last +weekday::operator[](last_spec) const NOEXCEPT +{ + return weekday_last{*this}; +} + +// year_month + +CONSTCD11 +inline +year_month::year_month(const date::year& y, const date::month& m) NOEXCEPT + : y_(y) + , m_(m) + {} + +CONSTCD11 inline year year_month::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month::month() const NOEXCEPT {return m_;} +CONSTCD11 inline bool year_month::ok() const NOEXCEPT {return y_.ok() && m_.ok();} + +CONSTCD14 +inline +year_month& +year_month::operator+=(const months& dm) NOEXCEPT +{ + *this = *this + dm; + return *this; +} + +CONSTCD14 +inline +year_month& +year_month::operator-=(const months& dm) NOEXCEPT +{ + *this = *this - dm; + return *this; +} + +CONSTCD14 +inline +year_month& +year_month::operator+=(const years& dy) NOEXCEPT +{ + *this = *this + dy; + return *this; +} + +CONSTCD14 +inline +year_month& +year_month::operator-=(const years& dy) NOEXCEPT +{ + *this = *this - dy; + return *this; +} + +CONSTCD11 +inline +bool +operator==(const year_month& x, const year_month& y) NOEXCEPT +{ + return x.year() == y.year() && x.month() == y.month(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month& x, const year_month& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const year_month& x, const year_month& y) NOEXCEPT +{ + return x.year() < y.year() ? true + : (x.year() > y.year() ? false + : (x.month() < y.month())); +} + +CONSTCD11 +inline +bool +operator>(const year_month& x, const year_month& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const year_month& x, const year_month& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const year_month& x, const year_month& y) NOEXCEPT +{ + return !(x < y); +} + +CONSTCD14 +inline +year_month +operator+(const year_month& ym, const months& dm) NOEXCEPT +{ + auto dmi = static_cast(static_cast(ym.month())) - 1 + dm.count(); + auto dy = (dmi >= 0 ? dmi : dmi-11) / 12; + dmi = dmi - dy * 12 + 1; + return (ym.year() + years(dy)) / month(static_cast(dmi)); +} + +CONSTCD14 +inline +year_month +operator+(const months& dm, const year_month& ym) NOEXCEPT +{ + return ym + dm; +} + +CONSTCD14 +inline +year_month +operator-(const year_month& ym, const months& dm) NOEXCEPT +{ + return ym + -dm; +} + +CONSTCD11 +inline +months +operator-(const year_month& x, const year_month& y) NOEXCEPT +{ + return (x.year() - y.year()) + + months(static_cast(x.month()) - static_cast(y.month())); +} + +CONSTCD11 +inline +year_month +operator+(const year_month& ym, const years& dy) NOEXCEPT +{ + return (ym.year() + dy) / ym.month(); +} + +CONSTCD11 +inline +year_month +operator+(const years& dy, const year_month& ym) NOEXCEPT +{ + return ym + dy; +} + +CONSTCD11 +inline +year_month +operator-(const year_month& ym, const years& dy) NOEXCEPT +{ + return ym + -dy; +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month& ym) +{ + return os << ym.year() << '/' << ym.month(); +} + +// month_day + +CONSTCD11 +inline +month_day::month_day(const date::month& m, const date::day& d) NOEXCEPT + : m_(m) + , d_(d) + {} + +CONSTCD11 inline date::month month_day::month() const NOEXCEPT {return m_;} +CONSTCD11 inline date::day month_day::day() const NOEXCEPT {return d_;} + +CONSTCD14 +inline +bool +month_day::ok() const NOEXCEPT +{ + CONSTDATA date::day d[] = + { + date::day(31), date::day(29), date::day(31), + date::day(30), date::day(31), date::day(30), + date::day(31), date::day(31), date::day(30), + date::day(31), date::day(30), date::day(31) + }; + return m_.ok() && date::day{1} <= d_ && d_ <= d[static_cast(m_)-1]; +} + +CONSTCD11 +inline +bool +operator==(const month_day& x, const month_day& y) NOEXCEPT +{ + return x.month() == y.month() && x.day() == y.day(); +} + +CONSTCD11 +inline +bool +operator!=(const month_day& x, const month_day& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const month_day& x, const month_day& y) NOEXCEPT +{ + return x.month() < y.month() ? true + : (x.month() > y.month() ? false + : (x.day() < y.day())); +} + +CONSTCD11 +inline +bool +operator>(const month_day& x, const month_day& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const month_day& x, const month_day& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const month_day& x, const month_day& y) NOEXCEPT +{ + return !(x < y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_day& md) +{ + return os << md.month() << '/' << md.day(); +} + +// month_day_last + +CONSTCD11 inline month month_day_last::month() const NOEXCEPT {return m_;} +CONSTCD11 inline bool month_day_last::ok() const NOEXCEPT {return m_.ok();} +CONSTCD11 inline month_day_last::month_day_last(const date::month& m) NOEXCEPT : m_(m) {} + +CONSTCD11 +inline +bool +operator==(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return x.month() == y.month(); +} + +CONSTCD11 +inline +bool +operator!=(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return x.month() < y.month(); +} + +CONSTCD11 +inline +bool +operator>(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const month_day_last& x, const month_day_last& y) NOEXCEPT +{ + return !(x < y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_day_last& mdl) +{ + return os << mdl.month() << "/last"; +} + +// month_weekday + +CONSTCD11 +inline +month_weekday::month_weekday(const date::month& m, + const date::weekday_indexed& wdi) NOEXCEPT + : m_(m) + , wdi_(wdi) + {} + +CONSTCD11 inline month month_weekday::month() const NOEXCEPT {return m_;} + +CONSTCD11 +inline +weekday_indexed +month_weekday::weekday_indexed() const NOEXCEPT +{ + return wdi_; +} + +CONSTCD11 +inline +bool +month_weekday::ok() const NOEXCEPT +{ + return m_.ok() && wdi_.ok(); +} + +CONSTCD11 +inline +bool +operator==(const month_weekday& x, const month_weekday& y) NOEXCEPT +{ + return x.month() == y.month() && x.weekday_indexed() == y.weekday_indexed(); +} + +CONSTCD11 +inline +bool +operator!=(const month_weekday& x, const month_weekday& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_weekday& mwd) +{ + return os << mwd.month() << '/' << mwd.weekday_indexed(); +} + +// month_weekday_last + +CONSTCD11 +inline +month_weekday_last::month_weekday_last(const date::month& m, + const date::weekday_last& wdl) NOEXCEPT + : m_(m) + , wdl_(wdl) + {} + +CONSTCD11 inline month month_weekday_last::month() const NOEXCEPT {return m_;} + +CONSTCD11 +inline +weekday_last +month_weekday_last::weekday_last() const NOEXCEPT +{ + return wdl_; +} + +CONSTCD11 +inline +bool +month_weekday_last::ok() const NOEXCEPT +{ + return m_.ok() && wdl_.ok(); +} + +CONSTCD11 +inline +bool +operator==(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT +{ + return x.month() == y.month() && x.weekday_last() == y.weekday_last(); +} + +CONSTCD11 +inline +bool +operator!=(const month_weekday_last& x, const month_weekday_last& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const month_weekday_last& mwdl) +{ + return os << mwdl.month() << '/' << mwdl.weekday_last(); +} + +// year_month_day_last + +CONSTCD11 +inline +year_month_day_last::year_month_day_last(const date::year& y, + const date::month_day_last& mdl) NOEXCEPT + : y_(y) + , mdl_(mdl) + {} + +CONSTCD14 +inline +year_month_day_last& +year_month_day_last::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +CONSTCD14 +inline +year_month_day_last& +year_month_day_last::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD14 +inline +year_month_day_last& +year_month_day_last::operator+=(const years& y) NOEXCEPT +{ + *this = *this + y; + return *this; +} + +CONSTCD14 +inline +year_month_day_last& +year_month_day_last::operator-=(const years& y) NOEXCEPT +{ + *this = *this - y; + return *this; +} + +CONSTCD11 inline year year_month_day_last::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month_day_last::month() const NOEXCEPT {return mdl_.month();} + +CONSTCD11 +inline +month_day_last +year_month_day_last::month_day_last() const NOEXCEPT +{ + return mdl_; +} + +CONSTCD14 +inline +day +year_month_day_last::day() const NOEXCEPT +{ + CONSTDATA date::day d[] = + { + date::day(31), date::day(28), date::day(31), + date::day(30), date::day(31), date::day(30), + date::day(31), date::day(31), date::day(30), + date::day(31), date::day(30), date::day(31) + }; + return month() != feb || !y_.is_leap() ? + d[static_cast(month()) - 1] : date::day{29}; +} + +CONSTCD14 +inline +year_month_day_last::operator sys_days() const NOEXCEPT +{ + return sys_days(year()/month()/day()); +} + +CONSTCD14 +inline +year_month_day_last::operator local_days() const NOEXCEPT +{ + return local_days(year()/month()/day()); +} + +CONSTCD11 +inline +bool +year_month_day_last::ok() const NOEXCEPT +{ + return y_.ok() && mdl_.ok(); +} + +CONSTCD11 +inline +bool +operator==(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return x.year() == y.year() && x.month_day_last() == y.month_day_last(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return x.year() < y.year() ? true + : (x.year() > y.year() ? false + : (x.month_day_last() < y.month_day_last())); +} + +CONSTCD11 +inline +bool +operator>(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const year_month_day_last& x, const year_month_day_last& y) NOEXCEPT +{ + return !(x < y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_day_last& ymdl) +{ + return os << ymdl.year() << '/' << ymdl.month_day_last(); +} + +CONSTCD14 +inline +year_month_day_last +operator+(const year_month_day_last& ymdl, const months& dm) NOEXCEPT +{ + return (ymdl.year() / ymdl.month() + dm) / last; +} + +CONSTCD14 +inline +year_month_day_last +operator+(const months& dm, const year_month_day_last& ymdl) NOEXCEPT +{ + return ymdl + dm; +} + +CONSTCD14 +inline +year_month_day_last +operator-(const year_month_day_last& ymdl, const months& dm) NOEXCEPT +{ + return ymdl + (-dm); +} + +CONSTCD11 +inline +year_month_day_last +operator+(const year_month_day_last& ymdl, const years& dy) NOEXCEPT +{ + return {ymdl.year()+dy, ymdl.month_day_last()}; +} + +CONSTCD11 +inline +year_month_day_last +operator+(const years& dy, const year_month_day_last& ymdl) NOEXCEPT +{ + return ymdl + dy; +} + +CONSTCD11 +inline +year_month_day_last +operator-(const year_month_day_last& ymdl, const years& dy) NOEXCEPT +{ + return ymdl + (-dy); +} + +// year_month_day + +CONSTCD11 +inline +year_month_day::year_month_day(const date::year& y, const date::month& m, + const date::day& d) NOEXCEPT + : y_(y) + , m_(m) + , d_(d) + {} + +CONSTCD14 +inline +year_month_day::year_month_day(const year_month_day_last& ymdl) NOEXCEPT + : y_(ymdl.year()) + , m_(ymdl.month()) + , d_(ymdl.day()) + {} + +CONSTCD14 +inline +year_month_day::year_month_day(sys_days dp) NOEXCEPT + : year_month_day(from_days(dp.time_since_epoch())) + {} + +CONSTCD14 +inline +year_month_day::year_month_day(local_days dp) NOEXCEPT + : year_month_day(from_days(dp.time_since_epoch())) + {} + +CONSTCD11 inline year year_month_day::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month_day::month() const NOEXCEPT {return m_;} +CONSTCD11 inline day year_month_day::day() const NOEXCEPT {return d_;} + +CONSTCD14 +inline +year_month_day& +year_month_day::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +CONSTCD14 +inline +year_month_day& +year_month_day::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD14 +inline +year_month_day& +year_month_day::operator+=(const years& y) NOEXCEPT +{ + *this = *this + y; + return *this; +} + +CONSTCD14 +inline +year_month_day& +year_month_day::operator-=(const years& y) NOEXCEPT +{ + *this = *this - y; + return *this; +} + +CONSTCD14 +inline +days +year_month_day::to_days() const NOEXCEPT +{ + static_assert(std::numeric_limits::digits >= 18, + "This algorithm has not been ported to a 16 bit unsigned integer"); + static_assert(std::numeric_limits::digits >= 20, + "This algorithm has not been ported to a 16 bit signed integer"); + auto const y = static_cast(y_) - (m_ <= feb); + auto const m = static_cast(m_); + auto const d = static_cast(d_); + auto const era = (y >= 0 ? y : y-399) / 400; + auto const yoe = static_cast(y - era * 400); // [0, 399] + auto const doy = (153*(m > 2 ? m-3 : m+9) + 2)/5 + d-1; // [0, 365] + auto const doe = yoe * 365 + yoe/4 - yoe/100 + doy; // [0, 146096] + return days{era * 146097 + static_cast(doe) - 719468}; +} + +CONSTCD14 +inline +year_month_day::operator sys_days() const NOEXCEPT +{ + return sys_days{to_days()}; +} + +CONSTCD14 +inline +year_month_day::operator local_days() const NOEXCEPT +{ + return local_days{to_days()}; +} + +CONSTCD14 +inline +bool +year_month_day::ok() const NOEXCEPT +{ + if (!(y_.ok() && m_.ok())) + return false; + return date::day{1} <= d_ && d_ <= (y_ / m_ / last).day(); +} + +CONSTCD11 +inline +bool +operator==(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return x.year() == y.year() && x.month() == y.month() && x.day() == y.day(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return !(x == y); +} + +CONSTCD11 +inline +bool +operator<(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return x.year() < y.year() ? true + : (x.year() > y.year() ? false + : (x.month() < y.month() ? true + : (x.month() > y.month() ? false + : (x.day() < y.day())))); +} + +CONSTCD11 +inline +bool +operator>(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return y < x; +} + +CONSTCD11 +inline +bool +operator<=(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return !(y < x); +} + +CONSTCD11 +inline +bool +operator>=(const year_month_day& x, const year_month_day& y) NOEXCEPT +{ + return !(x < y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_day& ymd) +{ + detail::save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os << ymd.year() << '-'; + os.width(2); + os << static_cast(ymd.month()) << '-'; + os << ymd.day(); + if (!ymd.ok()) + os << " is not a valid date"; + return os; +} + +CONSTCD14 +inline +year_month_day +year_month_day::from_days(days dp) NOEXCEPT +{ + static_assert(std::numeric_limits::digits >= 18, + "This algorithm has not been ported to a 16 bit unsigned integer"); + static_assert(std::numeric_limits::digits >= 20, + "This algorithm has not been ported to a 16 bit signed integer"); + auto const z = dp.count() + 719468; + auto const era = (z >= 0 ? z : z - 146096) / 146097; + auto const doe = static_cast(z - era * 146097); // [0, 146096] + auto const yoe = (doe - doe/1460 + doe/36524 - doe/146096) / 365; // [0, 399] + auto const y = static_cast(yoe) + era * 400; + auto const doy = doe - (365*yoe + yoe/4 - yoe/100); // [0, 365] + auto const mp = (5*doy + 2)/153; // [0, 11] + auto const d = doy - (153*mp+2)/5 + 1; // [1, 31] + auto const m = mp < 10 ? mp+3 : mp-9; // [1, 12] + return year_month_day{date::year{y + (m <= 2)}, date::month(m), date::day(d)}; +} + +CONSTCD14 +inline +year_month_day +operator+(const year_month_day& ymd, const months& dm) NOEXCEPT +{ + return (ymd.year() / ymd.month() + dm) / ymd.day(); +} + +CONSTCD14 +inline +year_month_day +operator+(const months& dm, const year_month_day& ymd) NOEXCEPT +{ + return ymd + dm; +} + +CONSTCD14 +inline +year_month_day +operator-(const year_month_day& ymd, const months& dm) NOEXCEPT +{ + return ymd + (-dm); +} + +CONSTCD11 +inline +year_month_day +operator+(const year_month_day& ymd, const years& dy) NOEXCEPT +{ + return (ymd.year() + dy) / ymd.month() / ymd.day(); +} + +CONSTCD11 +inline +year_month_day +operator+(const years& dy, const year_month_day& ymd) NOEXCEPT +{ + return ymd + dy; +} + +CONSTCD11 +inline +year_month_day +operator-(const year_month_day& ymd, const years& dy) NOEXCEPT +{ + return ymd + (-dy); +} + +// year_month_weekday + +CONSTCD11 +inline +year_month_weekday::year_month_weekday(const date::year& y, const date::month& m, + const date::weekday_indexed& wdi) + NOEXCEPT + : y_(y) + , m_(m) + , wdi_(wdi) + {} + +CONSTCD14 +inline +year_month_weekday::year_month_weekday(const sys_days& dp) NOEXCEPT + : year_month_weekday(from_days(dp.time_since_epoch())) + {} + +CONSTCD14 +inline +year_month_weekday::year_month_weekday(const local_days& dp) NOEXCEPT + : year_month_weekday(from_days(dp.time_since_epoch())) + {} + +CONSTCD14 +inline +year_month_weekday& +year_month_weekday::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +CONSTCD14 +inline +year_month_weekday& +year_month_weekday::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD14 +inline +year_month_weekday& +year_month_weekday::operator+=(const years& y) NOEXCEPT +{ + *this = *this + y; + return *this; +} + +CONSTCD14 +inline +year_month_weekday& +year_month_weekday::operator-=(const years& y) NOEXCEPT +{ + *this = *this - y; + return *this; +} + +CONSTCD11 inline year year_month_weekday::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month_weekday::month() const NOEXCEPT {return m_;} + +CONSTCD11 +inline +weekday +year_month_weekday::weekday() const NOEXCEPT +{ + return wdi_.weekday(); +} + +CONSTCD11 +inline +unsigned +year_month_weekday::index() const NOEXCEPT +{ + return wdi_.index(); +} + +CONSTCD11 +inline +weekday_indexed +year_month_weekday::weekday_indexed() const NOEXCEPT +{ + return wdi_; +} + +CONSTCD14 +inline +year_month_weekday::operator sys_days() const NOEXCEPT +{ + return sys_days{to_days()}; +} + +CONSTCD14 +inline +year_month_weekday::operator local_days() const NOEXCEPT +{ + return local_days{to_days()}; +} + +CONSTCD14 +inline +bool +year_month_weekday::ok() const NOEXCEPT +{ + if (!y_.ok() || !m_.ok() || !wdi_.weekday().ok() || wdi_.index() < 1) + return false; + if (wdi_.index() <= 4) + return true; + auto d2 = wdi_.weekday() - date::weekday(static_cast(y_/m_/1)) + days((wdi_.index()-1)*7 + 1); + return static_cast(d2.count()) <= static_cast((y_/m_/last).day()); +} + +CONSTCD14 +inline +year_month_weekday +year_month_weekday::from_days(days d) NOEXCEPT +{ + sys_days dp{d}; + auto const wd = date::weekday(dp); + auto const ymd = year_month_day(dp); + return {ymd.year(), ymd.month(), wd[(static_cast(ymd.day())-1)/7+1]}; +} + +CONSTCD14 +inline +days +year_month_weekday::to_days() const NOEXCEPT +{ + auto d = sys_days(y_/m_/1); + return (d + (wdi_.weekday() - date::weekday(d) + days{(wdi_.index()-1)*7}) + ).time_since_epoch(); +} + +CONSTCD11 +inline +bool +operator==(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT +{ + return x.year() == y.year() && x.month() == y.month() && + x.weekday_indexed() == y.weekday_indexed(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month_weekday& x, const year_month_weekday& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_weekday& ymwdi) +{ + return os << ymwdi.year() << '/' << ymwdi.month() + << '/' << ymwdi.weekday_indexed(); +} + +CONSTCD14 +inline +year_month_weekday +operator+(const year_month_weekday& ymwd, const months& dm) NOEXCEPT +{ + return (ymwd.year() / ymwd.month() + dm) / ymwd.weekday_indexed(); +} + +CONSTCD14 +inline +year_month_weekday +operator+(const months& dm, const year_month_weekday& ymwd) NOEXCEPT +{ + return ymwd + dm; +} + +CONSTCD14 +inline +year_month_weekday +operator-(const year_month_weekday& ymwd, const months& dm) NOEXCEPT +{ + return ymwd + (-dm); +} + +CONSTCD11 +inline +year_month_weekday +operator+(const year_month_weekday& ymwd, const years& dy) NOEXCEPT +{ + return {ymwd.year()+dy, ymwd.month(), ymwd.weekday_indexed()}; +} + +CONSTCD11 +inline +year_month_weekday +operator+(const years& dy, const year_month_weekday& ymwd) NOEXCEPT +{ + return ymwd + dy; +} + +CONSTCD11 +inline +year_month_weekday +operator-(const year_month_weekday& ymwd, const years& dy) NOEXCEPT +{ + return ymwd + (-dy); +} + +// year_month_weekday_last + +CONSTCD11 +inline +year_month_weekday_last::year_month_weekday_last(const date::year& y, + const date::month& m, + const date::weekday_last& wdl) NOEXCEPT + : y_(y) + , m_(m) + , wdl_(wdl) + {} + +CONSTCD14 +inline +year_month_weekday_last& +year_month_weekday_last::operator+=(const months& m) NOEXCEPT +{ + *this = *this + m; + return *this; +} + +CONSTCD14 +inline +year_month_weekday_last& +year_month_weekday_last::operator-=(const months& m) NOEXCEPT +{ + *this = *this - m; + return *this; +} + +CONSTCD14 +inline +year_month_weekday_last& +year_month_weekday_last::operator+=(const years& y) NOEXCEPT +{ + *this = *this + y; + return *this; +} + +CONSTCD14 +inline +year_month_weekday_last& +year_month_weekday_last::operator-=(const years& y) NOEXCEPT +{ + *this = *this - y; + return *this; +} + +CONSTCD11 inline year year_month_weekday_last::year() const NOEXCEPT {return y_;} +CONSTCD11 inline month year_month_weekday_last::month() const NOEXCEPT {return m_;} + +CONSTCD11 +inline +weekday +year_month_weekday_last::weekday() const NOEXCEPT +{ + return wdl_.weekday(); +} + +CONSTCD11 +inline +weekday_last +year_month_weekday_last::weekday_last() const NOEXCEPT +{ + return wdl_; +} + +CONSTCD14 +inline +year_month_weekday_last::operator sys_days() const NOEXCEPT +{ + return sys_days{to_days()}; +} + +CONSTCD14 +inline +year_month_weekday_last::operator local_days() const NOEXCEPT +{ + return local_days{to_days()}; +} + +CONSTCD11 +inline +bool +year_month_weekday_last::ok() const NOEXCEPT +{ + return y_.ok() && m_.ok() && wdl_.ok(); +} + +CONSTCD14 +inline +days +year_month_weekday_last::to_days() const NOEXCEPT +{ + auto const d = sys_days(y_/m_/last); + return (d - (date::weekday{d} - wdl_.weekday())).time_since_epoch(); +} + +CONSTCD11 +inline +bool +operator==(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT +{ + return x.year() == y.year() && x.month() == y.month() && + x.weekday_last() == y.weekday_last(); +} + +CONSTCD11 +inline +bool +operator!=(const year_month_weekday_last& x, const year_month_weekday_last& y) NOEXCEPT +{ + return !(x == y); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const year_month_weekday_last& ymwdl) +{ + return os << ymwdl.year() << '/' << ymwdl.month() << '/' << ymwdl.weekday_last(); +} + +CONSTCD14 +inline +year_month_weekday_last +operator+(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT +{ + return (ymwdl.year() / ymwdl.month() + dm) / ymwdl.weekday_last(); +} + +CONSTCD14 +inline +year_month_weekday_last +operator+(const months& dm, const year_month_weekday_last& ymwdl) NOEXCEPT +{ + return ymwdl + dm; +} + +CONSTCD14 +inline +year_month_weekday_last +operator-(const year_month_weekday_last& ymwdl, const months& dm) NOEXCEPT +{ + return ymwdl + (-dm); +} + +CONSTCD11 +inline +year_month_weekday_last +operator+(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT +{ + return {ymwdl.year()+dy, ymwdl.month(), ymwdl.weekday_last()}; +} + +CONSTCD11 +inline +year_month_weekday_last +operator+(const years& dy, const year_month_weekday_last& ymwdl) NOEXCEPT +{ + return ymwdl + dy; +} + +CONSTCD11 +inline +year_month_weekday_last +operator-(const year_month_weekday_last& ymwdl, const years& dy) NOEXCEPT +{ + return ymwdl + (-dy); +} + +// year_month from operator/() + +CONSTCD11 +inline +year_month +operator/(const year& y, const month& m) NOEXCEPT +{ + return {y, m}; +} + +CONSTCD11 +inline +year_month +operator/(const year& y, int m) NOEXCEPT +{ + return y / month(static_cast(m)); +} + +// month_day from operator/() + +CONSTCD11 +inline +month_day +operator/(const month& m, const day& d) NOEXCEPT +{ + return {m, d}; +} + +CONSTCD11 +inline +month_day +operator/(const day& d, const month& m) NOEXCEPT +{ + return m / d; +} + +CONSTCD11 +inline +month_day +operator/(const month& m, int d) NOEXCEPT +{ + return m / day(static_cast(d)); +} + +CONSTCD11 +inline +month_day +operator/(int m, const day& d) NOEXCEPT +{ + return month(static_cast(m)) / d; +} + +CONSTCD11 inline month_day operator/(const day& d, int m) NOEXCEPT {return m / d;} + +// month_day_last from operator/() + +CONSTCD11 +inline +month_day_last +operator/(const month& m, last_spec) NOEXCEPT +{ + return month_day_last{m}; +} + +CONSTCD11 +inline +month_day_last +operator/(last_spec, const month& m) NOEXCEPT +{ + return m/last; +} + +CONSTCD11 +inline +month_day_last +operator/(int m, last_spec) NOEXCEPT +{ + return month(static_cast(m))/last; +} + +CONSTCD11 +inline +month_day_last +operator/(last_spec, int m) NOEXCEPT +{ + return m/last; +} + +// month_weekday from operator/() + +CONSTCD11 +inline +month_weekday +operator/(const month& m, const weekday_indexed& wdi) NOEXCEPT +{ + return {m, wdi}; +} + +CONSTCD11 +inline +month_weekday +operator/(const weekday_indexed& wdi, const month& m) NOEXCEPT +{ + return m / wdi; +} + +CONSTCD11 +inline +month_weekday +operator/(int m, const weekday_indexed& wdi) NOEXCEPT +{ + return month(static_cast(m)) / wdi; +} + +CONSTCD11 +inline +month_weekday +operator/(const weekday_indexed& wdi, int m) NOEXCEPT +{ + return m / wdi; +} + +// month_weekday_last from operator/() + +CONSTCD11 +inline +month_weekday_last +operator/(const month& m, const weekday_last& wdl) NOEXCEPT +{ + return {m, wdl}; +} + +CONSTCD11 +inline +month_weekday_last +operator/(const weekday_last& wdl, const month& m) NOEXCEPT +{ + return m / wdl; +} + +CONSTCD11 +inline +month_weekday_last +operator/(int m, const weekday_last& wdl) NOEXCEPT +{ + return month(static_cast(m)) / wdl; +} + +CONSTCD11 +inline +month_weekday_last +operator/(const weekday_last& wdl, int m) NOEXCEPT +{ + return m / wdl; +} + +// year_month_day from operator/() + +CONSTCD11 +inline +year_month_day +operator/(const year_month& ym, const day& d) NOEXCEPT +{ + return {ym.year(), ym.month(), d}; +} + +CONSTCD11 +inline +year_month_day +operator/(const year_month& ym, int d) NOEXCEPT +{ + return ym / day(static_cast(d)); +} + +CONSTCD11 +inline +year_month_day +operator/(const year& y, const month_day& md) NOEXCEPT +{ + return y / md.month() / md.day(); +} + +CONSTCD11 +inline +year_month_day +operator/(int y, const month_day& md) NOEXCEPT +{ + return year(y) / md; +} + +CONSTCD11 +inline +year_month_day +operator/(const month_day& md, const year& y) NOEXCEPT +{ + return y / md; +} + +CONSTCD11 +inline +year_month_day +operator/(const month_day& md, int y) NOEXCEPT +{ + return year(y) / md; +} + +// year_month_day_last from operator/() + +CONSTCD11 +inline +year_month_day_last +operator/(const year_month& ym, last_spec) NOEXCEPT +{ + return {ym.year(), month_day_last{ym.month()}}; +} + +CONSTCD11 +inline +year_month_day_last +operator/(const year& y, const month_day_last& mdl) NOEXCEPT +{ + return {y, mdl}; +} + +CONSTCD11 +inline +year_month_day_last +operator/(int y, const month_day_last& mdl) NOEXCEPT +{ + return year(y) / mdl; +} + +CONSTCD11 +inline +year_month_day_last +operator/(const month_day_last& mdl, const year& y) NOEXCEPT +{ + return y / mdl; +} + +CONSTCD11 +inline +year_month_day_last +operator/(const month_day_last& mdl, int y) NOEXCEPT +{ + return year(y) / mdl; +} + +// year_month_weekday from operator/() + +CONSTCD11 +inline +year_month_weekday +operator/(const year_month& ym, const weekday_indexed& wdi) NOEXCEPT +{ + return {ym.year(), ym.month(), wdi}; +} + +CONSTCD11 +inline +year_month_weekday +operator/(const year& y, const month_weekday& mwd) NOEXCEPT +{ + return {y, mwd.month(), mwd.weekday_indexed()}; +} + +CONSTCD11 +inline +year_month_weekday +operator/(int y, const month_weekday& mwd) NOEXCEPT +{ + return year(y) / mwd; +} + +CONSTCD11 +inline +year_month_weekday +operator/(const month_weekday& mwd, const year& y) NOEXCEPT +{ + return y / mwd; +} + +CONSTCD11 +inline +year_month_weekday +operator/(const month_weekday& mwd, int y) NOEXCEPT +{ + return year(y) / mwd; +} + +// year_month_weekday_last from operator/() + +CONSTCD11 +inline +year_month_weekday_last +operator/(const year_month& ym, const weekday_last& wdl) NOEXCEPT +{ + return {ym.year(), ym.month(), wdl}; +} + +CONSTCD11 +inline +year_month_weekday_last +operator/(const year& y, const month_weekday_last& mwdl) NOEXCEPT +{ + return {y, mwdl.month(), mwdl.weekday_last()}; +} + +CONSTCD11 +inline +year_month_weekday_last +operator/(int y, const month_weekday_last& mwdl) NOEXCEPT +{ + return year(y) / mwdl; +} + +CONSTCD11 +inline +year_month_weekday_last +operator/(const month_weekday_last& mwdl, const year& y) NOEXCEPT +{ + return y / mwdl; +} + +CONSTCD11 +inline +year_month_weekday_last +operator/(const month_weekday_last& mwdl, int y) NOEXCEPT +{ + return year(y) / mwdl; +} + +template +struct fields; + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev = nullptr, + const std::chrono::seconds* offset_sec = nullptr); + +template +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr); + +// time_of_day + +enum {am = 1, pm}; + +namespace detail +{ + +// width::value is the number of fractional decimal digits in 1/n +// width<0>::value and width<1>::value are defined to be 0 +// If 1/n takes more than 18 fractional decimal digits, +// the result is truncated to 19. +// Example: width<2>::value == 1 +// Example: width<3>::value == 19 +// Example: width<4>::value == 2 +// Example: width<10>::value == 1 +// Example: width<1000>::value == 3 +template +struct width +{ + static CONSTDATA unsigned value = 1 + width::value; +}; + +template +struct width +{ + static CONSTDATA unsigned value = 0; +}; + +template +struct static_pow10 +{ +private: + static CONSTDATA std::uint64_t h = static_pow10::value; +public: + static CONSTDATA std::uint64_t value = h * h * (exp % 2 ? 10 : 1); +}; + +template <> +struct static_pow10<0> +{ + static CONSTDATA std::uint64_t value = 1; +}; + +template +struct make_precision +{ + using type = std::chrono::duration::value>>; + static CONSTDATA unsigned width = w; +}; + +template +struct make_precision +{ + using type = std::chrono::duration; + static CONSTDATA unsigned width = 6; +}; + +template ::type::period::den>::value> +class decimal_format_seconds +{ +public: + using rep = typename std::common_type::type::rep; + using precision = typename make_precision::type; + static auto CONSTDATA width = make_precision::width; + +private: + std::chrono::seconds s_; + precision sub_s_; + +public: + CONSTCD11 decimal_format_seconds() + : s_() + , sub_s_() + {} + + CONSTCD11 explicit decimal_format_seconds(const Duration& d) NOEXCEPT + : s_(std::chrono::duration_cast(d)) + , sub_s_(std::chrono::duration_cast(d - s_)) + {} + + CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT {return s_;} + CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT {return s_;} + CONSTCD11 precision subseconds() const NOEXCEPT {return sub_s_;} + + CONSTCD14 precision to_duration() const NOEXCEPT + { + return s_ + sub_s_; + } + + CONSTCD11 bool in_conventional_range() const NOEXCEPT + { + using namespace std::chrono; + return sub_s_ < std::chrono::seconds{1} && s_ < minutes{1}; + } + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, const decimal_format_seconds& x) + { + date::detail::save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << x.s_.count() << + std::use_facet>(os.getloc()).decimal_point(); + os.width(width); + os << static_cast(x.sub_s_.count()); + return os; + } +}; + +template +class decimal_format_seconds +{ + static CONSTDATA unsigned w = 0; +public: + using rep = typename std::common_type::type::rep; + using precision = std::chrono::duration; + static auto CONSTDATA width = make_precision::width; +private: + + std::chrono::seconds s_; + +public: + CONSTCD11 decimal_format_seconds() : s_() {} + CONSTCD11 explicit decimal_format_seconds(const precision& s) NOEXCEPT + : s_(s) + {} + + CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT {return s_;} + CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT {return s_;} + CONSTCD14 precision to_duration() const NOEXCEPT {return s_;} + + CONSTCD11 bool in_conventional_range() const NOEXCEPT + { + using namespace std::chrono; + return s_ < minutes{1}; + } + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, const decimal_format_seconds& x) + { + date::detail::save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << x.s_.count(); + return os; + } +}; + +enum class classify +{ + not_valid, + hour, + minute, + second, + subsecond +}; + +template +struct classify_duration +{ + static CONSTDATA classify value = + std::is_convertible::value + ? classify::hour : + std::is_convertible::value + ? classify::minute : + std::is_convertible::value + ? classify::second : + std::chrono::treat_as_floating_point::value + ? classify::not_valid : + classify::subsecond; +}; + +template +inline +CONSTCD11 +typename std::enable_if + < + std::numeric_limits::is_signed, + std::chrono::duration + >::type +abs(std::chrono::duration d) +{ + return d >= d.zero() ? d : -d; +} + +template +inline +CONSTCD11 +typename std::enable_if + < + !std::numeric_limits::is_signed, + std::chrono::duration + >::type +abs(std::chrono::duration d) +{ + return d; +} + +class time_of_day_base +{ +protected: + std::chrono::hours h_; + unsigned char mode_; + bool neg_; + + enum {is24hr}; + + CONSTCD11 time_of_day_base() NOEXCEPT + : h_(0) + , mode_(static_cast(is24hr)) + , neg_(false) + {} + + + CONSTCD11 time_of_day_base(std::chrono::hours h, bool neg, unsigned m) NOEXCEPT + : h_(detail::abs(h)) + , mode_(static_cast(m)) + , neg_(neg) + {} + + CONSTCD14 void make24() NOEXCEPT; + CONSTCD14 void make12() NOEXCEPT; + + CONSTCD14 std::chrono::hours to24hr() const; + + CONSTCD11 bool in_conventional_range() const NOEXCEPT + { + return !neg_ && h_ < days{1}; + } +}; + +CONSTCD14 +inline +std::chrono::hours +time_of_day_base::to24hr() const +{ + auto h = h_; + if (mode_ == am || mode_ == pm) + { + CONSTDATA auto h12 = std::chrono::hours(12); + if (mode_ == pm) + { + if (h != h12) + h = h + h12; + } + else if (h == h12) + h = std::chrono::hours(0); + } + return h; +} + +CONSTCD14 +inline +void +time_of_day_base::make24() NOEXCEPT +{ + h_ = to24hr(); + mode_ = is24hr; +} + +CONSTCD14 +inline +void +time_of_day_base::make12() NOEXCEPT +{ + if (mode_ == is24hr) + { + CONSTDATA auto h12 = std::chrono::hours(12); + if (h_ >= h12) + { + if (h_ > h12) + h_ = h_ - h12; + mode_ = pm; + } + else + { + if (h_ == std::chrono::hours(0)) + h_ = h12; + mode_ = am; + } + } +} + +template ::value> +class time_of_day_storage; + +template +class time_of_day_storage, detail::classify::hour> + : private detail::time_of_day_base +{ + using base = detail::time_of_day_base; + +public: + using precision = std::chrono::hours; + +#if !defined(_MSC_VER) || _MSC_VER >= 1900 + CONSTCD11 time_of_day_storage() NOEXCEPT = default; +#else + CONSTCD11 time_of_day_storage() = default; +#endif /* !defined(_MSC_VER) || _MSC_VER >= 1900 */ + + CONSTCD11 explicit time_of_day_storage(std::chrono::hours since_midnight) NOEXCEPT + : base(since_midnight, since_midnight < std::chrono::hours{0}, is24hr) + {} + + CONSTCD11 explicit time_of_day_storage(std::chrono::hours h, unsigned md) NOEXCEPT + : base(h, h < std::chrono::hours{0}, md) + {} + + CONSTCD11 std::chrono::hours hours() const NOEXCEPT {return h_;} + CONSTCD11 unsigned mode() const NOEXCEPT {return mode_;} + + CONSTCD14 explicit operator precision() const NOEXCEPT + { + auto p = to24hr(); + if (neg_) + p = -p; + return p; + } + + CONSTCD14 precision to_duration() const NOEXCEPT + { + return static_cast(*this); + } + + CONSTCD14 time_of_day_storage& make24() NOEXCEPT {base::make24(); return *this;} + CONSTCD14 time_of_day_storage& make12() NOEXCEPT {base::make12(); return *this;} + + CONSTCD11 bool in_conventional_range() const NOEXCEPT + { + return base::in_conventional_range(); + } + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, const time_of_day_storage& t) + { + using namespace std; + detail::save_stream _(os); + if (t.neg_) + os << '-'; + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + if (t.mode_ != am && t.mode_ != pm) + os.width(2); + os << t.h_.count(); + switch (t.mode_) + { + case time_of_day_storage::is24hr: + os << "00"; + break; + case am: + os << "am"; + break; + case pm: + os << "pm"; + break; + } + return os; + } +}; + +template +class time_of_day_storage, detail::classify::minute> + : private detail::time_of_day_base +{ + using base = detail::time_of_day_base; + + std::chrono::minutes m_; + +public: + using precision = std::chrono::minutes; + + CONSTCD11 time_of_day_storage() NOEXCEPT + : base() + , m_(0) + {} + + CONSTCD11 explicit time_of_day_storage(std::chrono::minutes since_midnight) NOEXCEPT + : base(std::chrono::duration_cast(since_midnight), + since_midnight < std::chrono::minutes{0}, is24hr) + , m_(detail::abs(since_midnight) - h_) + {} + + CONSTCD11 explicit time_of_day_storage(std::chrono::hours h, std::chrono::minutes m, + unsigned md) NOEXCEPT + : base(h, false, md) + , m_(m) + {} + + CONSTCD11 std::chrono::hours hours() const NOEXCEPT {return h_;} + CONSTCD11 std::chrono::minutes minutes() const NOEXCEPT {return m_;} + CONSTCD11 unsigned mode() const NOEXCEPT {return mode_;} + + CONSTCD14 explicit operator precision() const NOEXCEPT + { + auto p = to24hr() + m_; + if (neg_) + p = -p; + return p; + } + + CONSTCD14 precision to_duration() const NOEXCEPT + { + return static_cast(*this); + } + + CONSTCD14 time_of_day_storage& make24() NOEXCEPT {base::make24(); return *this;} + CONSTCD14 time_of_day_storage& make12() NOEXCEPT {base::make12(); return *this;} + + CONSTCD11 bool in_conventional_range() const NOEXCEPT + { + return base::in_conventional_range() && m_ < std::chrono::hours{1}; + } + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, const time_of_day_storage& t) + { + using namespace std; + detail::save_stream _(os); + if (t.neg_) + os << '-'; + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + if (t.mode_ != am && t.mode_ != pm) + os.width(2); + os << t.h_.count() << ':'; + os.width(2); + os << t.m_.count(); + switch (t.mode_) + { + case am: + os << "am"; + break; + case pm: + os << "pm"; + break; + } + return os; + } +}; + +template +class time_of_day_storage, detail::classify::second> + : private detail::time_of_day_base +{ + using base = detail::time_of_day_base; + using dfs = decimal_format_seconds; + + std::chrono::minutes m_; + dfs s_; + +public: + using precision = std::chrono::seconds; + + CONSTCD11 time_of_day_storage() NOEXCEPT + : base() + , m_(0) + , s_() + {} + + CONSTCD11 explicit time_of_day_storage(std::chrono::seconds since_midnight) NOEXCEPT + : base(std::chrono::duration_cast(since_midnight), + since_midnight < std::chrono::seconds{0}, is24hr) + , m_(std::chrono::duration_cast(detail::abs(since_midnight) - h_)) + , s_(detail::abs(since_midnight) - h_ - m_) + {} + + CONSTCD11 explicit time_of_day_storage(std::chrono::hours h, std::chrono::minutes m, + std::chrono::seconds s, unsigned md) NOEXCEPT + : base(h, false, md) + , m_(m) + , s_(s) + {} + + CONSTCD11 std::chrono::hours hours() const NOEXCEPT {return h_;} + CONSTCD11 std::chrono::minutes minutes() const NOEXCEPT {return m_;} + CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT {return s_.seconds();} + CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT {return s_.seconds();} + CONSTCD11 unsigned mode() const NOEXCEPT {return mode_;} + + CONSTCD14 explicit operator precision() const NOEXCEPT + { + auto p = to24hr() + s_.to_duration() + m_; + if (neg_) + p = -p; + return p; + } + + CONSTCD14 precision to_duration() const NOEXCEPT + { + return static_cast(*this); + } + + CONSTCD14 time_of_day_storage& make24() NOEXCEPT {base::make24(); return *this;} + CONSTCD14 time_of_day_storage& make12() NOEXCEPT {base::make12(); return *this;} + + CONSTCD11 bool in_conventional_range() const NOEXCEPT + { + return base::in_conventional_range() && m_ < std::chrono::hours{1} && + s_.in_conventional_range(); + } + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, const time_of_day_storage& t) + { + using namespace std; + detail::save_stream _(os); + if (t.neg_) + os << '-'; + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + if (t.mode_ != am && t.mode_ != pm) + os.width(2); + os << t.h_.count() << ':'; + os.width(2); + os << t.m_.count() << ':' << t.s_; + switch (t.mode_) + { + case am: + os << "am"; + break; + case pm: + os << "pm"; + break; + } + return os; + } + + template + friend + std::basic_ostream& + date::to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev, + const std::chrono::seconds* offset_sec); + + template + friend + std::basic_istream& + date::from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, + std::basic_string* abbrev, std::chrono::minutes* offset); +}; + +template +class time_of_day_storage, detail::classify::subsecond> + : private detail::time_of_day_base +{ +public: + using Duration = std::chrono::duration; + using dfs = decimal_format_seconds::type>; + using precision = typename dfs::precision; + +private: + using base = detail::time_of_day_base; + + std::chrono::minutes m_; + dfs s_; + +public: + CONSTCD11 time_of_day_storage() NOEXCEPT + : base() + , m_(0) + , s_() + {} + + CONSTCD11 explicit time_of_day_storage(Duration since_midnight) NOEXCEPT + : base(date::trunc(since_midnight), + since_midnight < Duration{0}, is24hr) + , m_(date::trunc(detail::abs(since_midnight) - h_)) + , s_(detail::abs(since_midnight) - h_ - m_) + {} + + CONSTCD11 explicit time_of_day_storage(std::chrono::hours h, std::chrono::minutes m, + std::chrono::seconds s, precision sub_s, + unsigned md) NOEXCEPT + : base(h, false, md) + , m_(m) + , s_(s + sub_s) + {} + + CONSTCD11 std::chrono::hours hours() const NOEXCEPT {return h_;} + CONSTCD11 std::chrono::minutes minutes() const NOEXCEPT {return m_;} + CONSTCD14 std::chrono::seconds& seconds() NOEXCEPT {return s_.seconds();} + CONSTCD11 std::chrono::seconds seconds() const NOEXCEPT {return s_.seconds();} + CONSTCD11 precision subseconds() const NOEXCEPT {return s_.subseconds();} + CONSTCD11 unsigned mode() const NOEXCEPT {return mode_;} + + CONSTCD14 explicit operator precision() const NOEXCEPT + { + auto p = to24hr() + s_.to_duration() + m_; + if (neg_) + p = -p; + return p; + } + + CONSTCD14 precision to_duration() const NOEXCEPT + { + return static_cast(*this); + } + + CONSTCD14 time_of_day_storage& make24() NOEXCEPT {base::make24(); return *this;} + CONSTCD14 time_of_day_storage& make12() NOEXCEPT {base::make12(); return *this;} + + CONSTCD11 bool in_conventional_range() const NOEXCEPT + { + return base::in_conventional_range() && m_ < std::chrono::hours{1} && + s_.in_conventional_range(); + } + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, const time_of_day_storage& t) + { + using namespace std; + detail::save_stream _(os); + if (t.neg_) + os << '-'; + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + if (t.mode_ != am && t.mode_ != pm) + os.width(2); + os << t.h_.count() << ':'; + os.width(2); + os << t.m_.count() << ':' << t.s_; + switch (t.mode_) + { + case am: + os << "am"; + break; + case pm: + os << "pm"; + break; + } + return os; + } + + template + friend + std::basic_ostream& + date::to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev, + const std::chrono::seconds* offset_sec); + + template + friend + std::basic_istream& + date::from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, + std::basic_string* abbrev, std::chrono::minutes* offset); +}; + +} // namespace detail + +template +class time_of_day + : public detail::time_of_day_storage +{ + using base = detail::time_of_day_storage; +public: +#if !defined(_MSC_VER) || _MSC_VER >= 1900 + CONSTCD11 time_of_day() NOEXCEPT = default; +#else + CONSTCD11 time_of_day() = default; +#endif /* !defined(_MSC_VER) || _MSC_VER >= 1900 */ + + CONSTCD11 explicit time_of_day(Duration since_midnight) NOEXCEPT + : base(since_midnight) + {} + + template + CONSTCD11 + explicit time_of_day(Arg0&& arg0, Arg1&& arg1, Args&& ...args) NOEXCEPT + : base(std::forward(arg0), std::forward(arg1), std::forward(args)...) + {} +}; + +template ::value>::type> +CONSTCD11 +inline +time_of_day> +make_time(const std::chrono::duration& d) +{ + return time_of_day>(d); +} + +CONSTCD11 +inline +time_of_day +make_time(const std::chrono::hours& h, unsigned md) +{ + return time_of_day(h, md); +} + +CONSTCD11 +inline +time_of_day +make_time(const std::chrono::hours& h, const std::chrono::minutes& m, + unsigned md) +{ + return time_of_day(h, m, md); +} + +CONSTCD11 +inline +time_of_day +make_time(const std::chrono::hours& h, const std::chrono::minutes& m, + const std::chrono::seconds& s, unsigned md) +{ + return time_of_day(h, m, s, md); +} + +template >::value>::type> +CONSTCD11 +inline +time_of_day> +make_time(const std::chrono::hours& h, const std::chrono::minutes& m, + const std::chrono::seconds& s, const std::chrono::duration& sub_s, + unsigned md) +{ + return time_of_day>(h, m, s, sub_s, md); +} + +template +inline +typename std::enable_if +< + !std::chrono::treat_as_floating_point::value && + std::ratio_less::value + , std::basic_ostream& +>::type +operator<<(std::basic_ostream& os, const sys_time& tp) +{ + auto const dp = date::floor(tp); + return os << year_month_day(dp) << ' ' << make_time(tp-dp); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const sys_days& dp) +{ + return os << year_month_day(dp); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const local_time& ut) +{ + return (os << sys_time{ut.time_since_epoch()}); +} + +// to_stream + +template +struct fields +{ + year_month_day ymd{year{0}/0/0}; + weekday wd{7u}; + time_of_day tod{}; + + fields() = default; + + fields(year_month_day ymd_) : ymd(ymd_) {} + fields(weekday wd_) : wd(wd_) {} + fields(time_of_day tod_) : tod(tod_) {} + + fields(year_month_day ymd_, weekday wd_) : ymd(ymd_), wd(wd_) {} + fields(year_month_day ymd_, time_of_day tod_) : ymd(ymd_), tod(tod_) {} + + fields(weekday wd_, time_of_day tod_) : wd(wd_), tod(tod_) {} + + fields(year_month_day ymd_, weekday wd_, time_of_day tod_) + : ymd(ymd_) + , wd(wd_) + , tod(tod_) + {} +}; + +namespace detail +{ + +template +unsigned +extract_weekday(std::basic_ostream& os, const fields& fds) +{ + if (!fds.ymd.ok() && !fds.wd.ok()) + { + // fds does not contain a valid weekday + os.setstate(std::ios::failbit); + return 7; + } + unsigned wd; + if (fds.ymd.ok()) + { + wd = static_cast(weekday{fds.ymd}); + if (fds.wd.ok() && wd != static_cast(fds.wd)) + { + // fds.ymd and fds.wd are inconsistent + os.setstate(std::ios::failbit); + return 7; + } + } + else + wd = static_cast(fds.wd); + return wd; +} + +template +unsigned +extract_month(std::basic_ostream& os, const fields& fds) +{ + if (!fds.ymd.month().ok()) + { + // fds does not contain a valid month + os.setstate(std::ios::failbit); + return 0; + } + return static_cast(fds.ymd.month()); +} + +} // namespace detail + +#if ONLY_C_LOCALE + +namespace detail +{ + +inline +std::pair +weekday_names() +{ + using namespace std; + static const string nm[] = + { + "Sunday", + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", + "Sun", + "Mon", + "Tue", + "Wed", + "Thu", + "Fri", + "Sat" + }; + return make_pair(nm, nm+sizeof(nm)/sizeof(nm[0])); +} + +inline +std::pair +month_names() +{ + using namespace std; + static const string nm[] = + { + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec" + }; + return make_pair(nm, nm+sizeof(nm)/sizeof(nm[0])); +} + +inline +std::pair +ampm_names() +{ + using namespace std; + static const string nm[] = + { + "AM", + "PM" + }; + return make_pair(nm, nm+sizeof(nm)/sizeof(nm[0])); +} + +template +FwdIter +scan_keyword(std::basic_istream& is, FwdIter kb, FwdIter ke) +{ + using namespace std; + size_t nkw = static_cast(std::distance(kb, ke)); + const unsigned char doesnt_match = '\0'; + const unsigned char might_match = '\1'; + const unsigned char does_match = '\2'; + unsigned char statbuf[100]; + unsigned char* status = statbuf; + unique_ptr stat_hold(0, free); + if (nkw > sizeof(statbuf)) + { + status = (unsigned char*)malloc(nkw); + if (status == nullptr) + throw bad_alloc(); + stat_hold.reset(status); + } + size_t n_might_match = nkw; // At this point, any keyword might match + size_t n_does_match = 0; // but none of them definitely do + // Initialize all statuses to might_match, except for "" keywords are does_match + unsigned char* st = status; + for (auto ky = kb; ky != ke; ++ky, ++st) + { + if (!ky->empty()) + *st = might_match; + else + { + *st = does_match; + --n_might_match; + ++n_does_match; + } + } + // While there might be a match, test keywords against the next CharT + for (size_t indx = 0; is && n_might_match > 0; ++indx) + { + // Peek at the next CharT but don't consume it + auto ic = is.peek(); + if (ic == EOF) + { + is.setstate(ios::eofbit); + break; + } + auto c = static_cast(toupper(ic)); + bool consume = false; + // For each keyword which might match, see if the indx character is c + // If a match if found, consume c + // If a match is found, and that is the last character in the keyword, + // then that keyword matches. + // If the keyword doesn't match this character, then change the keyword + // to doesn't match + st = status; + for (auto ky = kb; ky != ke; ++ky, ++st) + { + if (*st == might_match) + { + if (c == static_cast(toupper((*ky)[indx]))) + { + consume = true; + if (ky->size() == indx+1) + { + *st = does_match; + --n_might_match; + ++n_does_match; + } + } + else + { + *st = doesnt_match; + --n_might_match; + } + } + } + // consume if we matched a character + if (consume) + { + (void)is.get(); + // If we consumed a character and there might be a matched keyword that + // was marked matched on a previous iteration, then such keywords + // are now marked as not matching. + if (n_might_match + n_does_match > 1) + { + st = status; + for (auto ky = kb; ky != ke; ++ky, ++st) + { + if (*st == does_match && ky->size() != indx+1) + { + *st = doesnt_match; + --n_does_match; + } + } + } + } + } + // We've exited the loop because we hit eof and/or we have no more "might matches". + // Return the first matching result + for (st = status; kb != ke; ++kb, ++st) + if (*st == does_match) + break; + if (kb == ke) + is.setstate(ios_base::failbit); + return kb; +} + +} // namespace detail + +#endif // ONLY_C_LOCALE + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const fields& fds, const std::string* abbrev, + const std::chrono::seconds* offset_sec) +{ + using namespace std; + using namespace std::chrono; + using namespace detail; + tm tm{}; +#if !ONLY_C_LOCALE + auto& facet = use_facet>(os.getloc()); +#endif + const CharT* command = nullptr; + CharT modified = CharT{}; + for (; *fmt; ++fmt) + { + switch (*fmt) + { + case 'a': + case 'A': + if (command) + { + if (modified == CharT{}) + { + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; +#if !ONLY_C_LOCALE + const CharT f[] = {'%', *fmt}; + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); +#else // ONLY_C_LOCALE + os << weekday_names().first[tm.tm_wday+7*(*fmt == 'a')]; +#endif // ONLY_C_LOCALE + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'b': + case 'B': + case 'h': + if (command) + { + if (modified == CharT{}) + { + tm.tm_mon = static_cast(extract_month(os, fds)) - 1; +#if !ONLY_C_LOCALE + const CharT f[] = {'%', *fmt}; + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); +#else // ONLY_C_LOCALE + os << month_names().first[tm.tm_mon+12*(*fmt == 'b')]; +#endif // ONLY_C_LOCALE + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'c': + case 'x': + if (command) + { + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else + { +#if !ONLY_C_LOCALE + tm = std::tm{}; + auto const& ymd = fds.ymd; + auto ld = local_days(ymd); + tm.tm_sec = static_cast(fds.tod.seconds().count()); + tm.tm_min = static_cast(fds.tod.minutes().count()); + tm.tm_hour = static_cast(fds.tod.hours().count()); + tm.tm_mday = static_cast(static_cast(ymd.day())); + tm.tm_mon = static_cast(extract_month(os, fds) - 1); + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); + CharT f[3] = {'%'}; + auto fe = begin(f) + 1; + if (modified == CharT{'E'}) + *fe++ = modified; + *fe++ = *fmt; + facet.put(os, os, os.fill(), &tm, begin(f), fe); +#else // ONLY_C_LOCALE + if (*fmt == 'c') + { + auto wd = static_cast(extract_weekday(os, fds)); + os << weekday_names().first[static_cast(wd)+7] + << ' '; + os << month_names().first[extract_month(os, fds)-1+12] << ' '; + auto d = static_cast(static_cast(fds.ymd.day())); + if (d < 10) + os << ' '; + os << d << ' ' + << make_time(duration_cast(fds.tod.to_duration())) + << ' ' << fds.ymd.year(); + + } + else // *fmt == 'x' + { + auto const& ymd = fds.ymd; + save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << static_cast(ymd.month()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.day()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.year()) % 100; + } +#endif // ONLY_C_LOCALE + } + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'C': + if (command) + { + auto y = static_cast(fds.ymd.year()); +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + if (y >= 0) + { + os.width(2); + os << y/100; + } + else + { + os << CharT{'-'}; + os.width(2); + os << -(y-99)/100; + } +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'E'}) + { + tm.tm_year = y - 1900; + CharT f[3] = {'%', 'E', 'C'}; + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'd': + case 'e': + if (command) + { + auto d = static_cast(static_cast(fds.ymd.day())); +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + save_stream _(os); + if (*fmt == CharT{'d'}) + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << d; +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + tm.tm_mday = d; + CharT f[3] = {'%', 'O', *fmt}; + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'D': + if (command) + { + if (modified == CharT{}) + { + auto const& ymd = fds.ymd; + save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << static_cast(ymd.month()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.day()) << CharT{'/'}; + os.width(2); + os << static_cast(ymd.year()) % 100; + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'F': + if (command) + { + if (modified == CharT{}) + { + auto const& ymd = fds.ymd; + save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(4); + os << static_cast(ymd.year()) << CharT{'-'}; + os.width(2); + os << static_cast(ymd.month()) << CharT{'-'}; + os.width(2); + os << static_cast(ymd.day()); + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'g': + case 'G': + if (command) + { + if (modified == CharT{}) + { + auto ld = local_days(fds.ymd); + auto y = year_month_day{ld + days{3}}.year(); + auto start = local_days((y - years{1})/date::dec/thu[last]) + (mon-thu); + if (ld < start) + --y; + if (*fmt == CharT{'G'}) + os << y; + else + { + save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(2); + os << std::abs(static_cast(y)) % 100; + } + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'H': + case 'I': + if (command) + { + auto hms = fds.tod; +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + if (*fmt == CharT{'I'}) + hms.make12(); + if (hms.hours() < hours{10}) + os << CharT{'0'}; + os << hms.hours().count(); +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_hour = static_cast(hms.hours().count()); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'j': + if (command) + { + if (modified == CharT{}) + { + auto ld = local_days(fds.ymd); + auto y = fds.ymd.year(); + auto doy = ld - local_days(y/jan/1) + days{1}; + save_stream _(os); + os.fill('0'); + os.flags(std::ios::dec | std::ios::right); + os.width(3); + os << doy.count(); + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'm': + if (command) + { + auto m = static_cast(fds.ymd.month()); +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + if (m < 10) + os << CharT{'0'}; + os << m; +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_mon = static_cast(m-1); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'M': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + if (fds.tod.minutes() < minutes{10}) + os << CharT{'0'}; + os << fds.tod.minutes().count(); +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_min = static_cast(fds.tod.minutes().count()); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'n': + if (command) + { + if (modified == CharT{}) + os << CharT{'\n'}; + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'p': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { + const CharT f[] = {'%', *fmt}; + tm.tm_hour = static_cast(fds.tod.hours().count()); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#else + if (fds.tod.hours() < hours{12}) + os << ampm_names().first[0]; + else + os << ampm_names().first[1]; +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'r': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { + const CharT f[] = {'%', *fmt}; + tm.tm_hour = static_cast(fds.tod.hours().count()); + tm.tm_min = static_cast(fds.tod.minutes().count()); + tm.tm_sec = static_cast(fds.tod.seconds().count()); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#else + time_of_day tod(duration_cast(fds.tod.to_duration())); + tod.make12(); + save_stream _(os); + os.fill('0'); + os.width(2); + os << tod.hours().count() << CharT{':'}; + os.width(2); + os << tod.minutes().count() << CharT{':'}; + os.width(2); + os << tod.seconds().count() << CharT{' '}; + tod.make24(); + if (tod.hours() < hours{12}) + os << ampm_names().first[0]; + else + os << ampm_names().first[1]; +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'R': + if (command) + { + if (modified == CharT{}) + { + if (fds.tod.hours() < hours{10}) + os << CharT{'0'}; + os << fds.tod.hours().count() << CharT{':'}; + if (fds.tod.minutes() < minutes{10}) + os << CharT{'0'}; + os << fds.tod.minutes().count(); + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'S': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + os << fds.tod.s_; +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_sec = static_cast(fds.tod.s_.seconds().count()); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 't': + if (command) + { + if (modified == CharT{}) + os << CharT{'\t'}; + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'T': + if (command) + { + if (modified == CharT{}) + { + os << fds.tod; + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'u': + if (command) + { + auto wd = extract_weekday(os, fds); + if (os.fail()) + return os; +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + os << (wd != 0 ? wd : 7u); +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_wday = static_cast(wd); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'U': + if (command) + { + auto const& ymd = fds.ymd; + auto ld = local_days(ymd); +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + auto st = local_days(sun[1]/jan/ymd.year()); + if (ld < st) + os << CharT{'0'} << CharT{'0'}; + else + { + auto wn = duration_cast(ld - st).count() + 1; + if (wn < 10) + os << CharT{'0'}; + os << wn; + } + #if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'V': + if (command) + { + auto ld = local_days(fds.ymd); +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + auto y = year_month_day{ld + days{3}}.year(); + auto st = local_days((y - years{1})/12/thu[last]) + (mon-thu); + if (ld < st) + { + --y; + st = local_days((y - years{1})/12/thu[last]) + (mon-thu); + } + auto wn = duration_cast(ld - st).count() + 1; + if (wn < 10) + os << CharT{'0'}; + os << wn; +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + auto const& ymd = fds.ymd; + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'w': + if (command) + { + auto wd = extract_weekday(os, fds); + if (os.fail()) + return os; +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + os << wd; +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_wday = static_cast(wd); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'W': + if (command) + { + auto const& ymd = fds.ymd; + auto ld = local_days(ymd); +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + auto st = local_days(mon[1]/jan/ymd.year()); + if (ld < st) + os << CharT{'0'} << CharT{'0'}; + else + { + auto wn = duration_cast(ld - st).count() + 1; + if (wn < 10) + os << CharT{'0'}; + os << wn; + } +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = static_cast(ymd.year()) - 1900; + tm.tm_wday = static_cast(extract_weekday(os, fds)); + if (os.fail()) + return os; + tm.tm_yday = static_cast((ld - local_days(ymd.year()/1/1)).count()); + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'X': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{'O'}) + os << CharT{'%'} << modified << *fmt; + else + { + tm = std::tm{}; + tm.tm_sec = static_cast(fds.tod.seconds().count()); + tm.tm_min = static_cast(fds.tod.minutes().count()); + tm.tm_hour = static_cast(fds.tod.hours().count()); + CharT f[3] = {'%'}; + auto fe = begin(f) + 1; + if (modified == CharT{'E'}) + *fe++ = modified; + *fe++ = *fmt; + facet.put(os, os, os.fill(), &tm, begin(f), fe); + } +#else + os << fds.tod; +#endif + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'y': + if (command) + { + auto y = static_cast(fds.ymd.year()); +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + y = std::abs(y) % 100; + if (y < 10) + os << CharT{'0'}; + os << y; +#if !ONLY_C_LOCALE + } + else + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = y - 1900; + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'Y': + if (command) + { + auto y = fds.ymd.year(); +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + os << y; +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'E'}) + { + const CharT f[] = {'%', modified, *fmt}; + tm.tm_year = static_cast(y) - 1900; + facet.put(os, os, os.fill(), &tm, begin(f), end(f)); + } + else + { + os << CharT{'%'} << modified << *fmt; + } +#endif + modified = CharT{}; + command = nullptr; + } + else + os << *fmt; + break; + case 'z': + if (command) + { + if (offset_sec == nullptr) + { + // Can not format %z with unknown offset + os.setstate(ios::failbit); + return os; + } + auto m = duration_cast(*offset_sec); + auto neg = m < minutes{0}; + m = date::abs(m); + auto h = duration_cast(m); + m -= h; + if (neg) + os << CharT{'-'}; + else + os << CharT{'+'}; + if (h < hours{10}) + os << CharT{'0'}; + os << h.count(); + if (modified != CharT{}) + os << CharT{':'}; + if (m < minutes{10}) + os << CharT{'0'}; + os << m.count(); + command = nullptr; + modified = CharT{}; + } + else + os << *fmt; + break; + case 'Z': + if (command) + { + if (modified == CharT{}) + { + if (abbrev == nullptr) + { + // Can not format %Z with unknown time_zone + os.setstate(ios::failbit); + return os; + } + for (auto c : *abbrev) + os << CharT(c); + } + else + { + os << CharT{'%'} << modified << *fmt; + modified = CharT{}; + } + command = nullptr; + } + else + os << *fmt; + break; + case 'E': + case 'O': + if (command) + { + if (modified == CharT{}) + { + modified = *fmt; + } + else + { + os << CharT{'%'} << modified << *fmt; + command = nullptr; + modified = CharT{}; + } + } + else + os << *fmt; + break; + case '%': + if (command) + { + if (modified == CharT{}) + { + os << CharT{'%'}; + command = nullptr; + } + else + { + os << CharT{'%'} << modified << CharT{'%'}; + command = nullptr; + modified = CharT{}; + } + } + else + command = fmt; + break; + default: + if (command) + { + os << CharT{'%'}; + command = nullptr; + } + if (modified != CharT{}) + { + os << modified; + modified = CharT{}; + } + os << *fmt; + break; + } + } + if (command) + os << CharT{'%'}; + if (modified != CharT{}) + os << modified; + return os; +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const year& y) +{ + using CT = std::chrono::seconds; + fields fds{y/0/0}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const month& m) +{ + using CT = std::chrono::seconds; + fields fds{m/0/0}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const day& d) +{ + using CT = std::chrono::seconds; + fields fds{d/0/0}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const weekday& wd) +{ + using CT = std::chrono::seconds; + fields fds{wd}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const year_month& ym) +{ + using CT = std::chrono::seconds; + fields fds{ym/0}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, const month_day& md) +{ + using CT = std::chrono::seconds; + fields fds{md/0}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const year_month_day& ymd) +{ + using CT = std::chrono::seconds; + fields fds{ymd}; + return to_stream(os, fmt, fds); +} + +template +inline +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const std::chrono::duration& d) +{ + using Duration = std::chrono::duration; + using CT = typename std::common_type::type; + fields fds{time_of_day{d}}; + return to_stream(os, fmt, fds); +} + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const local_time& tp, const std::string* abbrev = nullptr, + const std::chrono::seconds* offset_sec = nullptr) +{ + using CT = typename std::common_type::type; + auto ld = floor(tp); + fields fds{year_month_day{ld}, time_of_day{tp-local_seconds{ld}}}; + return to_stream(os, fmt, fds, abbrev, offset_sec); +} + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const sys_time& tp) +{ + using namespace std::chrono; + using CT = typename std::common_type::type; + const std::string abbrev("UTC"); + CONSTDATA seconds offset{0}; + auto sd = floor(tp); + fields fds{year_month_day{sd}, time_of_day{tp-sys_seconds{sd}}}; + return to_stream(os, fmt, fds, &abbrev, &offset); +} + +// format + +template +auto +format(const std::locale& loc, const CharT* fmt, const Streamable& tp) + -> decltype(to_stream(std::declval&>(), fmt, tp), + std::basic_string{}) +{ + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + os.imbue(loc); + to_stream(os, fmt, tp); + return os.str(); +} + +template +auto +format(const CharT* fmt, const Streamable& tp) + -> decltype(to_stream(std::declval&>(), fmt, tp), + std::basic_string{}) +{ + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + to_stream(os, fmt, tp); + return os.str(); +} + +template +auto +format(const std::locale& loc, const std::basic_string& fmt, + const Streamable& tp) + -> decltype(to_stream(std::declval&>(), fmt.c_str(), tp), + std::basic_string{}) +{ + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + os.imbue(loc); + to_stream(os, fmt.c_str(), tp); + return os.str(); +} + +template +auto +format(const std::basic_string& fmt, const Streamable& tp) + -> decltype(to_stream(std::declval&>(), fmt.c_str(), tp), + std::basic_string{}) +{ + std::basic_ostringstream os; + os.exceptions(std::ios::failbit | std::ios::badbit); + to_stream(os, fmt.c_str(), tp); + return os.str(); +} + +// parse + +namespace detail +{ + +template +bool +read_char(std::basic_istream& is, CharT fmt, std::ios::iostate& err) +{ + auto ic = is.get(); + if (Traits::eq_int_type(ic, Traits::eof()) || + !Traits::eq(Traits::to_char_type(ic), fmt)) + { + err |= std::ios::failbit; + is.setstate(std::ios::failbit); + return false; + } + return true; +} + +template +unsigned +read_unsigned(std::basic_istream& is, unsigned m = 1, unsigned M = 10) +{ + unsigned x = 0; + unsigned count = 0; + while (true) + { + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + break; + auto c = static_cast(Traits::to_char_type(ic)); + if (!('0' <= c && c <= '9')) + break; + (void)is.get(); + ++count; + x = 10*x + static_cast(c - '0'); + if (count == M) + break; + } + if (count < m) + is.setstate(std::ios::failbit); + return x; +} + +template +int +read_signed(std::basic_istream& is, unsigned m = 1, unsigned M = 10) +{ + auto ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) + { + auto c = static_cast(Traits::to_char_type(ic)); + if (('0' <= c && c <= '9') || c == '-' || c == '+') + { + if (c == '-' || c == '+') + (void)is.get(); + auto x = static_cast(read_unsigned(is, std::max(m, 1u), M)); + if (!is.fail()) + { + if (c == '-') + x = -x; + return x; + } + } + } + if (m > 0) + is.setstate(std::ios::failbit); + return 0; +} + +template +long double +read_long_double(std::basic_istream& is, unsigned m = 1, unsigned M = 10) +{ + using namespace std; + unsigned count = 0; + auto decimal_point = Traits::to_int_type( + use_facet>(is.getloc()).decimal_point()); + std::string buf; + while (true) + { + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + break; + if (Traits::eq_int_type(ic, decimal_point)) + { + buf += '.'; + decimal_point = Traits::eof(); + is.get(); + } + else + { + auto c = static_cast(Traits::to_char_type(ic)); + if (!('0' <= c && c <= '9')) + break; + buf += c; + (void)is.get(); + } + if (++count == M) + break; + } + if (count < m) + { + is.setstate(std::ios::failbit); + return 0; + } + return std::stold(buf); +} + +struct rs +{ + int& i; + unsigned m; + unsigned M; +}; + +struct ru +{ + int& i; + unsigned m; + unsigned M; +}; + +struct rld +{ + long double& i; + unsigned m; + unsigned M; +}; + +template +void +read(std::basic_istream&) +{ +} + +template +void +read(std::basic_istream& is, CharT a0, Args&& ...args); + +template +void +read(std::basic_istream& is, rs a0, Args&& ...args); + +template +void +read(std::basic_istream& is, ru a0, Args&& ...args); + +template +void +read(std::basic_istream& is, int a0, Args&& ...args); + +template +void +read(std::basic_istream& is, rld a0, Args&& ...args); + +template +void +read(std::basic_istream& is, CharT a0, Args&& ...args) +{ + // No-op if a0 == CharT{} + if (a0 != CharT{}) + { + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + { + is.setstate(std::ios::failbit | std::ios::eofbit); + return; + } + if (!Traits::eq(Traits::to_char_type(ic), a0)) + { + is.setstate(std::ios::failbit); + return; + } + (void)is.get(); + } + read(is, std::forward(args)...); +} + +template +void +read(std::basic_istream& is, rs a0, Args&& ...args) +{ + auto x = read_signed(is, a0.m, a0.M); + if (is.fail()) + return; + a0.i = x; + read(is, std::forward(args)...); +} + +template +void +read(std::basic_istream& is, ru a0, Args&& ...args) +{ + auto x = read_unsigned(is, a0.m, a0.M); + if (is.fail()) + return; + a0.i = static_cast(x); + read(is, std::forward(args)...); +} + +template +void +read(std::basic_istream& is, int a0, Args&& ...args) +{ + if (a0 != -1) + { + auto u = static_cast(a0); + CharT buf[std::numeric_limits::digits10+2] = {}; + auto e = buf; + do + { + *e++ = CharT(u % 10) + CharT{'0'}; + u /= 10; + } while (u > 0); + std::reverse(buf, e); + for (auto p = buf; p != e && is.rdstate() == std::ios::goodbit; ++p) + read(is, *p); + } + if (is.rdstate() == std::ios::goodbit) + read(is, std::forward(args)...); +} + +template +void +read(std::basic_istream& is, rld a0, Args&& ...args) +{ + auto x = read_long_double(is, a0.m, a0.M); + if (is.fail()) + return; + a0.i = x; + read(is, std::forward(args)...); +} + +} // namespace detail; + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + fields& fds, std::basic_string* abbrev, + std::chrono::minutes* offset) +{ + using namespace std; + using namespace std::chrono; + typename basic_istream::sentry ok{is, true}; + if (ok) + { +#if !ONLY_C_LOCALE + auto& f = use_facet>(is.getloc()); + std::tm tm{}; +#endif + std::basic_string temp_abbrev; + minutes temp_offset{}; + const CharT* command = nullptr; + auto modified = CharT{}; + auto width = -1; + CONSTDATA int not_a_year = numeric_limits::min(); + int Y = not_a_year; + CONSTDATA int not_a_century = not_a_year / 100; + int C = not_a_century; + CONSTDATA int not_a_2digit_year = 100; + int y = not_a_2digit_year; + int m{}; + int d{}; + int j{}; + CONSTDATA int not_a_weekday = 7; + int wd = not_a_weekday; + CONSTDATA int not_a_hour_12_value = 0; + int I = not_a_hour_12_value; + hours h{}; + minutes min{}; + Duration s{}; + int g = not_a_2digit_year; + int G = not_a_year; + CONSTDATA int not_a_week_num = 100; + int V = not_a_week_num; + int U = not_a_week_num; + int W = not_a_week_num; + using detail::read; + using detail::rs; + using detail::ru; + using detail::rld; + for (; *fmt && is.rdstate() == std::ios::goodbit; ++fmt) + { + switch (*fmt) + { + case 'a': + case 'A': + if (command) + { +#if !ONLY_C_LOCALE + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + wd = tm.tm_wday; + is.setstate(err); +#else + auto nm = detail::weekday_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (!is.fail()) + wd = i % 7; +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'b': + case 'B': + case 'h': + if (command) + { +#if !ONLY_C_LOCALE + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + m = tm.tm_mon + 1; + is.setstate(err); +#else + auto nm = detail::month_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (!is.fail()) + m = i % 12 + 1; +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'c': + if (command) + { +#if !ONLY_C_LOCALE + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + Y = tm.tm_year + 1900; + m = tm.tm_mon + 1; + d = tm.tm_mday; + h = hours{tm.tm_hour}; + min = minutes{tm.tm_min}; + s = duration_cast(seconds{tm.tm_sec}); + } + is.setstate(err); +#else + auto nm = detail::weekday_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (is.fail()) + goto broken; + wd = i % 7; + ws(is); + nm = detail::month_names(); + i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (is.fail()) + goto broken; + m = i % 12 + 1; + ws(is); + read(is, rs{d, 1, 2}); + if (is.fail()) + goto broken; + ws(is); + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int H; + int M; + long double S; + read(is, ru{H, 1, 2}, CharT{':'}, ru{M, 1, 2}, + CharT{':'}, rld{S, 1, w}); + if (is.fail()) + goto broken; + h = hours{H}; + min = minutes{M}; + s = round(duration{S}); + ws(is); + read(is, rs{Y, 1, 4u}); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'x': + if (command) + { +#if !ONLY_C_LOCALE + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + Y = tm.tm_year + 1900; + m = tm.tm_mon + 1; + d = tm.tm_mday; + } + is.setstate(err); +#else + read(is, ru{m, 1, 2}, CharT{'/'}, ru{d, 1, 2}, CharT{'/'}, + rs{y, 1, 2}); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'X': + if (command) + { +#if !ONLY_C_LOCALE + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + h = hours{tm.tm_hour}; + min = minutes{tm.tm_min}; + s = duration_cast(seconds{tm.tm_sec}); + } + is.setstate(err); +#else + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int H; + int M; + long double S; + read(is, ru{H, 1, 2}, CharT{':'}, ru{M, 1, 2}, + CharT{':'}, rld{S, 1, w}); + if (!is.fail()) + { + h = hours{H}; + min = minutes{M}; + s = round(duration{S}); + } +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'C': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + read(is, rs{C, 1, width == -1 ? 2u : static_cast(width)}); +#if !ONLY_C_LOCALE + } + else + { + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + auto tY = tm.tm_year + 1900; + C = (tY >= 0 ? tY : tY-99) / 100; + } + is.setstate(err); + } +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'D': + if (command) + { + if (modified == CharT{}) + read(is, ru{m, 1, 2}, CharT{'\0'}, CharT{'/'}, CharT{'\0'}, + ru{d, 1, 2}, CharT{'\0'}, CharT{'/'}, CharT{'\0'}, + rs{y, 1, 2}); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'F': + if (command) + { + if (modified == CharT{}) + read(is, rs{Y, 1, width == -1 ? 4u : static_cast(width)}, + CharT{'-'}, ru{m, 1, 2}, CharT{'-'}, ru{d, 1, 2}); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'd': + case 'e': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + read(is, rs{d, 1, width == -1 ? 2u : static_cast(width)}); +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + command = nullptr; + width = -1; + modified = CharT{}; + if ((err & ios::failbit) == 0) + d = tm.tm_mday; + is.setstate(err); + } + else + read(is, CharT{'%'}, width, modified, *fmt); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'H': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + int H; + read(is, ru{H, 1, width == -1 ? 2u : static_cast(width)}); + if (!is.fail()) + h = hours{H}; +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + h = hours{tm.tm_hour}; + is.setstate(err); + } + else + read(is, CharT{'%'}, width, modified, *fmt); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'I': + if (command) + { + if (modified == CharT{}) + { + // reads in an hour into I, but most be in [1, 12] + read(is, rs{I, 1, width == -1 ? 2u : static_cast(width)}); + if (I != not_a_hour_12_value) + { + if (!(1 <= I && I <= 12)) + { + I = not_a_hour_12_value; + goto broken; + } + } + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'j': + if (command) + { + if (modified == CharT{}) + read(is, ru{j, 1, width == -1 ? 3u : static_cast(width)}); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'M': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + int M; + read(is, ru{M, 1, width == -1 ? 2u : static_cast(width)}); + if (!is.fail()) + min = minutes{M}; +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + min = minutes{tm.tm_min}; + is.setstate(err); + } + else + read(is, CharT{'%'}, width, modified, *fmt); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'm': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + read(is, rs{m, 1, width == -1 ? 2u : static_cast(width)}); +#if !ONLY_C_LOCALE + else if (modified == CharT{'O'}) + { + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + m = tm.tm_mon + 1; + is.setstate(err); + } + else + read(is, CharT{'%'}, width, modified, *fmt); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'n': + case 't': + if (command) + { + // %n matches a single white space character + // %t matches 0 or 1 white space characters + auto ic = is.peek(); + if (Traits::eq_int_type(ic, Traits::eof())) + { + ios_base::iostate err = ios_base::eofbit; + if (*fmt == 'n') + err |= ios_base::failbit; + is.setstate(err); + break; + } + if (isspace(ic)) + { + (void)is.get(); + } + else if (*fmt == 'n') + is.setstate(ios_base::failbit); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'p': + // Error if haven't yet seen %I + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { + if (I == not_a_hour_12_value) + goto broken; + tm = std::tm{}; + tm.tm_hour = I; + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if (err & ios::failbit) + goto broken; + h = hours{tm.tm_hour}; + I = not_a_hour_12_value; + } + else + read(is, CharT{'%'}, width, modified, *fmt); +#else + if (I == not_a_hour_12_value) + goto broken; + auto nm = detail::ampm_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (is.fail()) + goto broken; + h = hours{I}; + if (i == 1) + { + if (h != hours{12}) + h += hours{12}; + } + else if (h == hours{12}) + h = hours{0}; + I = not_a_hour_12_value; +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + + break; + case 'r': + if (command) + { +#if !ONLY_C_LOCALE + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + { + h = hours{tm.tm_hour}; + min = minutes{tm.tm_min}; + s = duration_cast(seconds{tm.tm_sec}); + } + is.setstate(err); +#else + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int H; + int M; + long double S; + read(is, ru{H, 1, 2}, CharT{':'}, ru{M, 1, 2}, + CharT{':'}, rld{S, 1, w}); + if (is.fail() || !(1 <= H && H <= 12)) + goto broken; + ws(is); + auto nm = detail::ampm_names(); + auto i = detail::scan_keyword(is, nm.first, nm.second) - nm.first; + if (is.fail()) + goto broken; + h = hours{H}; + if (i == 1) + { + if (h != hours{12}) + h += hours{12}; + } + else if (h == hours{12}) + h = hours{0}; + min = minutes{M}; + s = round(duration{S}); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'R': + if (command) + { + if (modified == CharT{}) + { + int H, M; + read(is, ru{H, 1, 2}, CharT{'\0'}, CharT{':'}, CharT{'\0'}, + ru{M, 1, 2}, CharT{'\0'}); + if (!is.fail()) + { + h = hours{H}; + min = minutes{M}; + } + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'S': + if (command) + { + #if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + long double S; + read(is, rld{S, 1, width == -1 ? w : static_cast(width)}); + if (!is.fail()) + s = round(duration{S}); +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + s = duration_cast(seconds{tm.tm_sec}); + is.setstate(err); + } + else + read(is, CharT{'%'}, width, modified, *fmt); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'T': + if (command) + { + if (modified == CharT{}) + { + using dfs = detail::decimal_format_seconds; + CONSTDATA auto w = Duration::period::den == 1 ? 2 : 3 + dfs::width; + int H; + int M; + long double S; + read(is, ru{H, 1, 2}, CharT{':'}, ru{M, 1, 2}, + CharT{':'}, rld{S, 1, w}); + if (!is.fail()) + { + h = hours{H}; + min = minutes{M}; + s = round(duration{S}); + } + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'Y': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + read(is, rs{Y, 1, width == -1 ? 4u : static_cast(width)}); +#if !ONLY_C_LOCALE + else if (modified == CharT{'E'}) + { + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + Y = tm.tm_year + 1900; + is.setstate(err); + } + else + read(is, CharT{'%'}, width, modified, *fmt); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'y': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) +#endif + read(is, ru{y, 1, width == -1 ? 2u : static_cast(width)}); +#if !ONLY_C_LOCALE + else + { + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + Y = tm.tm_year + 1900; + is.setstate(err); + } +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'g': + if (command) + { + if (modified == CharT{}) + read(is, ru{g, 1, width == -1 ? 2u : static_cast(width)}); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'G': + if (command) + { + if (modified == CharT{}) + read(is, rs{G, 1, width == -1 ? 4u : static_cast(width)}); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'U': + if (command) + { + if (modified == CharT{}) + read(is, ru{U, 1, width == -1 ? 2u : static_cast(width)}); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'V': + if (command) + { + if (modified == CharT{}) + read(is, ru{V, 1, width == -1 ? 2u : static_cast(width)}); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'W': + if (command) + { + if (modified == CharT{}) + read(is, ru{W, 1, width == -1 ? 2u : static_cast(width)}); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'u': + case 'w': + if (command) + { +#if !ONLY_C_LOCALE + if (modified == CharT{}) + { +#endif + read(is, ru{wd, 1, width == -1 ? 1u : static_cast(width)}); + if (!is.fail() && *fmt == 'u') + { + if (wd == 7) + wd = 0; + else if (wd == 0) + wd = 7; + } +#if !ONLY_C_LOCALE + } + else if (modified == CharT{'O'}) + { + ios_base::iostate err = ios_base::goodbit; + f.get(is, nullptr, is, err, &tm, command, fmt+1); + if ((err & ios::failbit) == 0) + wd = tm.tm_wday; + is.setstate(err); + } + else + read(is, CharT{'%'}, width, modified, *fmt); +#endif + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'E': + case 'O': + if (command) + { + if (modified == CharT{}) + { + modified = *fmt; + } + else + { + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + } + else + read(is, *fmt); + break; + case '%': + if (command) + { + if (modified == CharT{}) + read(is, *fmt); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + command = fmt; + break; + case 'z': + if (command) + { + int H, M; + if (modified == CharT{}) + { + read(is, rs{H, 2, 2}); + if (!is.fail()) + temp_offset = hours{H}; + if (is.good()) + { + auto ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) + { + auto c = static_cast(Traits::to_char_type(ic)); + if ('0' <= c && c <= '9') + { + read(is, ru{M, 2, 2}); + if (!is.fail()) + temp_offset += minutes{ H < 0 ? -M : M }; + } + } + } + } + else + { + read(is, rs{H, 1, 2}); + if (!is.fail()) + temp_offset = hours{H}; + if (is.good()) + { + auto ic = is.peek(); + if (!Traits::eq_int_type(ic, Traits::eof())) + { + auto c = static_cast(Traits::to_char_type(ic)); + if (c == ':') + { + (void)is.get(); + read(is, ru{M, 2, 2}); + if (!is.fail()) + temp_offset += minutes{ H < 0 ? -M : M }; + } + } + } + } + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + case 'Z': + if (command) + { + if (modified == CharT{}) + { + if (!temp_abbrev.empty()) + is.setstate(ios::failbit); + else + { + while (is.rdstate() == std::ios::goodbit) + { + auto i = is.rdbuf()->sgetc(); + if (Traits::eq_int_type(i, Traits::eof())) + { + is.setstate(ios::eofbit); + break; + } + auto wc = Traits::to_char_type(i); + auto c = static_cast(wc); + // is c a valid time zone name or abbreviation character? + if (!(CharT{1} < wc && wc < CharT{127}) || !(isalnum(c) || + c == '_' || c == '/' || c == '-' || c == '+')) + break; + temp_abbrev.push_back(c); + is.rdbuf()->sbumpc(); + } + if (temp_abbrev.empty()) + is.setstate(ios::failbit); + } + } + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + else + read(is, *fmt); + break; + default: + if (command) + { + if (width == -1 && modified == CharT{} && '0' <= *fmt && *fmt <= '9') + { + width = static_cast(*fmt) - '0'; + while ('0' <= fmt[1] && fmt[1] <= '9') + width = 10*width + static_cast(*++fmt) - '0'; + } + else + { + if (modified == CharT{}) + read(is, CharT{'%'}, width, *fmt); + else + read(is, CharT{'%'}, width, modified, *fmt); + command = nullptr; + width = -1; + modified = CharT{}; + } + } + else // !command + { + if (isspace(*fmt)) + ws(is); // space matches 0 or more white space characters + else + read(is, *fmt); + } + break; + } + } + // is.rdstate() != ios::goodbit || *fmt == CharT{} + if (is.rdstate() == ios::goodbit && command) + { + if (modified == CharT{}) + read(is, CharT{'%'}, width); + else + read(is, CharT{'%'}, width, modified); + } + if (is.rdstate() != ios::goodbit && *fmt != CharT{} && !is.fail()) + is.setstate(ios::failbit); + if (!is.fail()) + { + if (y != not_a_2digit_year) + { + // Convert y and an optional C to Y + if (!(0 <= y && y <= 99)) + goto broken; + if (C == not_a_century) + { + if (Y == not_a_year) + { + if (y >= 69) + C = 19; + else + C = 20; + } + else + { + C = (Y >= 0 ? Y : Y-100) / 100; + } + } + int tY; + if (C >= 0) + tY = 100*C + y; + else + tY = 100*(C+1) - (y == 0 ? 100 : y); + if (Y != not_a_year && Y != tY) + goto broken; + Y = tY; + } + if (g != not_a_2digit_year) + { + // Convert g and an optional C to G + if (!(0 <= g && g <= 99)) + goto broken; + if (C == not_a_century) + { + if (G == not_a_year) + { + if (g >= 69) + C = 19; + else + C = 20; + } + else + { + C = (G >= 0 ? G : G-100) / 100; + } + } + int tG; + if (C >= 0) + tG = 100*C + g; + else + tG = 100*(C+1) - (g == 0 ? 100 : g); + if (G != not_a_year && G != tG) + goto broken; + G = tG; + } + if (G != not_a_year) + { + // Convert G, V and wd to Y, m and d + if (V == not_a_week_num || wd == not_a_weekday) + goto broken; + auto ymd = year_month_day{local_days(year{G-1}/dec/thu[last]) + + (mon-thu) + weeks{V-1} + + (weekday{static_cast(wd)}-mon)}; + if (Y == not_a_year) + Y = static_cast(ymd.year()); + else if (year{Y} != ymd.year()) + goto broken; + if (m == 0) + m = static_cast(static_cast(ymd.month())); + else if (month(static_cast(m)) != ymd.month()) + goto broken; + if (d == 0) + d = static_cast(static_cast(ymd.day())); + else if (day(static_cast(d)) != ymd.day()) + goto broken; + } + if (j != 0 && Y != not_a_year) + { + auto ymd = year_month_day{local_days(year{Y}/1/1) + days{j-1}}; + if (m == 0) + m = static_cast(static_cast(ymd.month())); + else if (month(static_cast(m)) != ymd.month()) + goto broken; + if (d == 0) + d = static_cast(static_cast(ymd.day())); + else if (day(static_cast(d)) != ymd.day()) + goto broken; + } + if (U != not_a_week_num && Y != not_a_year) + { + if (wd == not_a_weekday) + goto broken; + sys_days sd; + if (U == 0) + sd = year{Y-1}/dec/weekday{static_cast(wd)}[last]; + else + sd = sys_days(year{Y}/jan/sun[1]) + weeks{U-1} + + (weekday{static_cast(wd)} - sun); + year_month_day ymd = sd; + if (year{Y} != ymd.year()) + goto broken; + if (m == 0) + m = static_cast(static_cast(ymd.month())); + else if (month(static_cast(m)) != ymd.month()) + goto broken; + if (d == 0) + d = static_cast(static_cast(ymd.day())); + else if (day(static_cast(d)) != ymd.day()) + goto broken; + } + if (W != not_a_week_num && Y != not_a_year) + { + if (wd == not_a_weekday) + goto broken; + sys_days sd; + if (W == 0) + sd = year{Y-1}/dec/weekday{static_cast(wd)}[last]; + else + sd = sys_days(year{Y}/jan/mon[1]) + weeks{W-1} + + (weekday{static_cast(wd)} - mon); + year_month_day ymd = sd; + if (year{Y} != ymd.year()) + goto broken; + if (m == 0) + m = static_cast(static_cast(ymd.month())); + else if (month(static_cast(m)) != ymd.month()) + goto broken; + if (d == 0) + d = static_cast(static_cast(ymd.day())); + else if (day(static_cast(d)) != ymd.day()) + goto broken; + } + if (Y < static_cast(year::min()) || Y > static_cast(year::max())) + Y = not_a_year; + auto ymd = year{Y}/m/d; + if (wd != not_a_weekday && ymd.ok()) + { + if (weekday{static_cast(wd)} != weekday(ymd)) + goto broken; + } + fds.ymd = ymd; + fds.tod = time_of_day{h}; + fds.tod.m_ = min; + fds.tod.s_ = detail::decimal_format_seconds{s}; + if (wd != not_a_weekday) + fds.wd = weekday{static_cast(wd)}; + if (abbrev != nullptr) + *abbrev = std::move(temp_abbrev); + if (offset != nullptr) + *offset = temp_offset; + } + return is; + } +broken: + is.setstate(ios_base::failbit); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, year& y, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.year().ok()) + is.setstate(ios::failbit); + if (!is.fail()) + y = fds.ymd.year(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, month& m, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.month().ok()) + is.setstate(ios::failbit); + if (!is.fail()) + m = fds.ymd.month(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, day& d, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.day().ok()) + is.setstate(ios::failbit); + if (!is.fail()) + d = fds.ymd.day(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, weekday& wd, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.wd.ok()) + is.setstate(ios::failbit); + if (!is.fail()) + wd = fds.wd; + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, year_month& ym, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.month().ok()) + is.setstate(ios::failbit); + if (!is.fail()) + ym = fds.ymd.year()/fds.ymd.month(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, month_day& md, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.month().ok() || !fds.ymd.day().ok()) + is.setstate(ios::failbit); + if (!is.fail()) + md = fds.ymd.month()/fds.ymd.day(); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + year_month_day& ymd, std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = seconds; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.ok()) + is.setstate(ios::failbit); + if (!is.fail()) + ymd = fds.ymd; + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + sys_time& tp, std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = typename common_type::type; + minutes offset_local{}; + auto offptr = offset ? offset : &offset_local; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offptr); + if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) + is.setstate(ios::failbit); + if (!is.fail()) + tp = round(sys_days(fds.ymd) - *offptr + fds.tod.to_duration()); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + local_time& tp, std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = typename common_type::type; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) + is.setstate(ios::failbit); + if (!is.fail()) + tp = round(local_seconds{local_days(fds.ymd)} + fds.tod.to_duration()); + return is; +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + std::chrono::duration& d, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using Duration = std::chrono::duration; + using CT = typename common_type::type; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offset); + if (!is.fail()) + d = duration_cast(fds.tod.to_duration()); + return is; +} + +template , + class Alloc = std::allocator> +struct parse_manip +{ + const std::basic_string format_; + Parsable& tp_; + std::basic_string* abbrev_; + std::chrono::minutes* offset_; + +public: + parse_manip(std::basic_string format, Parsable& tp, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) + : format_(std::move(format)) + , tp_(tp) + , abbrev_(abbrev) + , offset_(offset) + {} + +}; + +template +std::basic_istream& +operator>>(std::basic_istream& is, + const parse_manip& x) +{ + return from_stream(is, x.format_.c_str(), x.tp_, x.abbrev_, x.offset_); +} + +template +inline +auto +parse(const std::basic_string& format, Parsable& tp) + -> decltype(from_stream(std::declval&>(), + format.c_str(), tp), + parse_manip{format, tp}) +{ + return {format, tp}; +} + +template +inline +auto +parse(const std::basic_string& format, Parsable& tp, + std::basic_string& abbrev) + -> decltype(from_stream(std::declval&>(), + format.c_str(), tp, &abbrev), + parse_manip{format, tp, &abbrev}) +{ + return {format, tp, &abbrev}; +} + +template +inline +auto +parse(const std::basic_string& format, Parsable& tp, + std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), + format.c_str(), tp, nullptr, &offset), + parse_manip{format, tp, nullptr, &offset}) +{ + return {format, tp, nullptr, &offset}; +} + +template +inline +auto +parse(const std::basic_string& format, Parsable& tp, + std::basic_string& abbrev, std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), + format.c_str(), tp, &abbrev, &offset), + parse_manip{format, tp, &abbrev, &offset}) +{ + return {format, tp, &abbrev, &offset}; +} + +// const CharT* formats + +template +inline +auto +parse(const CharT* format, Parsable& tp) + -> decltype(from_stream(std::declval&>(), format, tp), + parse_manip{format, tp}) +{ + return {format, tp}; +} + +template +inline +auto +parse(const CharT* format, Parsable& tp, std::basic_string& abbrev) + -> decltype(from_stream(std::declval&>(), format, + tp, &abbrev), + parse_manip{format, tp, &abbrev}) +{ + return {format, tp, &abbrev}; +} + +template +inline +auto +parse(const CharT* format, Parsable& tp, std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), format, + tp, nullptr, &offset), + parse_manip{format, tp, nullptr, &offset}) +{ + return {format, tp, nullptr, &offset}; +} + +template +inline +auto +parse(const CharT* format, Parsable& tp, + std::basic_string& abbrev, std::chrono::minutes& offset) + -> decltype(from_stream(std::declval&>(), format, + tp, &abbrev, &offset), + parse_manip{format, tp, &abbrev, &offset}) +{ + return {format, tp, &abbrev, &offset}; +} + +// duration streaming + +namespace detail +{ + +#if __cplusplus >= 201402 && (!defined(__EDG_VERSION__) || __EDG_VERSION__ > 411) \ + && (!defined(__SUNPRO_CC) || __SUNPRO_CC > 0x5150) + +template +class string_literal +{ + CharT p_[N]; + +public: + using const_iterator = const CharT*; + + string_literal(string_literal const&) = default; + string_literal& operator=(string_literal const&) = delete; + + template > + CONSTCD14 string_literal(CharT c) NOEXCEPT + : p_{c} + { + } + + CONSTCD14 string_literal(const CharT(&a)[N]) NOEXCEPT + : p_{} + { + for (std::size_t i = 0; i < N; ++i) + p_[i] = a[i]; + } + + template > + CONSTCD14 string_literal(const char(&a)[N]) NOEXCEPT + : p_{} + { + for (std::size_t i = 0; i < N; ++i) + p_[i] = a[i]; + } + + template {}>> + CONSTCD14 string_literal(string_literal const& a) NOEXCEPT + : p_{} + { + for (std::size_t i = 0; i < N; ++i) + p_[i] = a[i]; + } + + template > + CONSTCD14 string_literal(const string_literal& x, + const string_literal& y) NOEXCEPT + : p_{} + { + std::size_t i = 0; + for (; i < N1-1; ++i) + p_[i] = x[i]; + for (std::size_t j = 0; j < N2; ++j, ++i) + p_[i] = y[j]; + } + + CONSTCD14 const CharT* data() const NOEXCEPT {return p_;} + CONSTCD14 std::size_t size() const NOEXCEPT {return N-1;} + + CONSTCD14 const_iterator begin() const NOEXCEPT {return p_;} + CONSTCD14 const_iterator end() const NOEXCEPT {return p_ + N-1;} + + CONSTCD14 CharT const& operator[](std::size_t n) const NOEXCEPT + { + return p_[n]; + } + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, const string_literal& s) + { + return os << s.p_; + } +}; + +template +CONSTCD14 +inline +string_literal, + N1 + N2 - 1> +operator+(const string_literal& x, const string_literal& y) NOEXCEPT +{ + using CharT = std::conditional_t; + return string_literal{string_literal{x}, + string_literal{y}}; +} + +template +inline +std::basic_string +operator+(std::basic_string x, + const string_literal& y) NOEXCEPT +{ + x.append(y.data(), y.size()); + return x; +} + +template +CONSTCD14 +inline +string_literal +msl(const CharT(&a)[N]) NOEXCEPT +{ + return string_literal{a}; +} + +template {} || + std::is_same{} || + std::is_same{} || + std::is_same{}>> +CONSTCD14 +inline +string_literal +msl(CharT c) NOEXCEPT +{ + return string_literal{c}; +} + +CONSTCD14 +inline +std::size_t +to_string_len(std::intmax_t i) +{ + std::size_t r = 0; + do + { + i /= 10; + ++r; + } while (i > 0); + return r; +} + +template +CONSTCD14 +inline +std::enable_if_t +< + N < 10, + string_literal +> +msl() NOEXCEPT +{ + return msl(char(N % 10 + '0')); +} + +template +CONSTCD14 +inline +std::enable_if_t +< + 10 <= N, + string_literal +> +msl() NOEXCEPT +{ + return msl() + msl(char(N % 10 + '0')); +} + +template +CONSTCD14 +inline +std::enable_if_t +< + std::ratio::type::den != 1, + string_literal::type::num) + + to_string_len(std::ratio::type::den) + 4> +> +msl(std::ratio) NOEXCEPT +{ + using R = typename std::ratio::type; + return msl(CharT{'['}) + msl() + msl(CharT{'/'}) + + msl() + msl(CharT{']'}); +} + +template +CONSTCD14 +inline +std::enable_if_t +< + std::ratio::type::den == 1, + string_literal::type::num) + 3> +> +msl(std::ratio) NOEXCEPT +{ + using R = typename std::ratio::type; + return msl(CharT{'['}) + msl() + msl(CharT{']'}); +} + +template +CONSTCD14 +inline +auto +msl(std::atto) NOEXCEPT +{ + return msl(CharT{'a'}); +} + +template +CONSTCD14 +inline +auto +msl(std::femto) NOEXCEPT +{ + return msl(CharT{'f'}); +} + +template +CONSTCD14 +inline +auto +msl(std::pico) NOEXCEPT +{ + return msl(CharT{'p'}); +} + +template +CONSTCD14 +inline +auto +msl(std::nano) NOEXCEPT +{ + return msl(CharT{'n'}); +} + +template +CONSTCD14 +inline +std::enable_if_t +< + std::is_same{}, + string_literal +> +msl(std::micro) NOEXCEPT +{ + return string_literal{"\xC2\xB5"}; +} + +template +CONSTCD14 +inline +std::enable_if_t +< + !std::is_same{}, + string_literal +> +msl(std::micro) NOEXCEPT +{ + return string_literal{CharT{static_cast('\xB5')}}; +} + +template +CONSTCD14 +inline +auto +msl(std::milli) NOEXCEPT +{ + return msl(CharT{'m'}); +} + +template +CONSTCD14 +inline +auto +msl(std::centi) NOEXCEPT +{ + return msl(CharT{'c'}); +} + +template +CONSTCD14 +inline +auto +msl(std::deci) NOEXCEPT +{ + return msl(CharT{'d'}); +} + +template +CONSTCD14 +inline +auto +msl(std::deca) NOEXCEPT +{ + return string_literal{"da"}; +} + +template +CONSTCD14 +inline +auto +msl(std::hecto) NOEXCEPT +{ + return msl(CharT{'h'}); +} + +template +CONSTCD14 +inline +auto +msl(std::kilo) NOEXCEPT +{ + return msl(CharT{'k'}); +} + +template +CONSTCD14 +inline +auto +msl(std::mega) NOEXCEPT +{ + return msl(CharT{'M'}); +} + +template +CONSTCD14 +inline +auto +msl(std::giga) NOEXCEPT +{ + return msl(CharT{'G'}); +} + +template +CONSTCD14 +inline +auto +msl(std::tera) NOEXCEPT +{ + return msl(CharT{'T'}); +} + +template +CONSTCD14 +inline +auto +msl(std::peta) NOEXCEPT +{ + return msl(CharT{'P'}); +} + +template +CONSTCD14 +inline +auto +msl(std::exa) NOEXCEPT +{ + return msl(CharT{'E'}); +} + +template +CONSTCD14 +auto +get_units(Period p) +{ + return msl(p) + string_literal{"s"}; +} + +template +CONSTCD14 +auto +get_units(std::ratio<1>) +{ + return string_literal{"s"}; +} + +template +CONSTCD14 +auto +get_units(std::ratio<60>) +{ + return string_literal{"min"}; +} + +template +CONSTCD14 +auto +get_units(std::ratio<3600>) +{ + return string_literal{"h"}; +} + +#else // __cplusplus < 201402 || (defined(__EDG_VERSION__) && __EDG_VERSION__ <= 411) + +inline +std::string +to_string(std::uint64_t x) +{ + return std::to_string(x); +} + +template +std::basic_string +to_string(std::uint64_t x) +{ + auto y = std::to_string(x); + return std::basic_string(y.begin(), y.end()); +} + +template +inline +typename std::enable_if +< + std::ratio::type::den != 1, + std::basic_string +>::type +msl(std::ratio) +{ + using R = typename std::ratio::type; + return std::basic_string(1, '[') + to_string(R::num) + CharT{'/'} + + to_string(R::den) + CharT{']'}; +} + +template +inline +typename std::enable_if +< + std::ratio::type::den == 1, + std::basic_string +>::type +msl(std::ratio) +{ + using R = typename std::ratio::type; + return std::basic_string(1, '[') + to_string(R::num) + CharT{']'}; +} + +template +inline +std::basic_string +msl(std::atto) +{ + return {'a'}; +} + +template +inline +std::basic_string +msl(std::femto) +{ + return {'f'}; +} + +template +inline +std::basic_string +msl(std::pico) +{ + return {'p'}; +} + +template +inline +std::basic_string +msl(std::nano) +{ + return {'n'}; +} + +template +inline +typename std::enable_if +< + std::is_same::value, + std::string +>::type +msl(std::micro) +{ + return "\xC2\xB5"; +} + +template +inline +typename std::enable_if +< + !std::is_same::value, + std::basic_string +>::type +msl(std::micro) +{ + return {CharT(static_cast('\xB5'))}; +} + +template +inline +std::basic_string +msl(std::milli) +{ + return {'m'}; +} + +template +inline +std::basic_string +msl(std::centi) +{ + return {'c'}; +} + +template +inline +std::basic_string +msl(std::deci) +{ + return {'d'}; +} + +template +inline +std::basic_string +msl(std::deca) +{ + return {'d', 'a'}; +} + +template +inline +std::basic_string +msl(std::hecto) +{ + return {'h'}; +} + +template +inline +std::basic_string +msl(std::kilo) +{ + return {'k'}; +} + +template +inline +std::basic_string +msl(std::mega) +{ + return {'M'}; +} + +template +inline +std::basic_string +msl(std::giga) +{ + return {'G'}; +} + +template +inline +std::basic_string +msl(std::tera) +{ + return {'T'}; +} + +template +inline +std::basic_string +msl(std::peta) +{ + return {'P'}; +} + +template +inline +std::basic_string +msl(std::exa) +{ + return {'E'}; +} + +template +std::basic_string +get_units(Period p) +{ + return msl(p) + CharT{'s'}; +} + +template +std::basic_string +get_units(std::ratio<1>) +{ + return {'s'}; +} + +template +std::basic_string +get_units(std::ratio<60>) +{ + return {'m', 'i', 'n'}; +} + +template +std::basic_string +get_units(std::ratio<3600>) +{ + return {'h'}; +} + +#endif // __cplusplus < 201402 || (defined(__EDG_VERSION__) && __EDG_VERSION__ <= 411) + +template > +struct make_string; + +template <> +struct make_string +{ + template + static + std::string + from(Rep n) + { + return std::to_string(n); + } +}; + +template +struct make_string +{ + template + static + std::basic_string + from(Rep n) + { + auto s = std::to_string(n); + return std::basic_string(s.begin(), s.end()); + } +}; + +template <> +struct make_string +{ + template + static + std::wstring + from(Rep n) + { + return std::to_wstring(n); + } +}; + +template +struct make_string +{ + template + static + std::basic_string + from(Rep n) + { + auto s = std::to_wstring(n); + return std::basic_string(s.begin(), s.end()); + } +}; + +} // namespace detail + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, + const std::chrono::duration& d) +{ + using namespace detail; + return os << make_string::from(d.count()) + + get_units(typename Period::type{}); +} + +} // namespace date +} // namespace util +} // namespace arrow + + +#ifdef __GNUC__ +# pragma GCC diagnostic pop +#endif + + +#endif // DATE_H diff --git a/cpp/src/arrow/vendored/datetime/ios.h b/cpp/src/arrow/vendored/datetime/ios.h new file mode 100644 index 0000000000000..ec8342f55c9ef --- /dev/null +++ b/cpp/src/arrow/vendored/datetime/ios.h @@ -0,0 +1,56 @@ +// +// ios.h +// DateTimeLib +// +// The MIT License (MIT) +// +// Copyright (c) 2016 Alexander Kormanovsky +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ios_hpp +#define ios_hpp + +#if __APPLE__ +# include +# if TARGET_OS_IPHONE +# include + + namespace arrow + { + namespace util + { + namespace date + { + namespace iOSUtils + { + + std::string get_tzdata_path(); + std::string get_current_timezone(); + + } // namespace iOSUtils + } // namespace date + } // namespace util + } // namespace arrow + +# endif // TARGET_OS_IPHONE +#else // !__APPLE__ +# define TARGET_OS_IPHONE 0 +#endif // !__APPLE__ +#endif // ios_hpp diff --git a/cpp/src/arrow/vendored/datetime/tz.cpp b/cpp/src/arrow/vendored/datetime/tz.cpp new file mode 100644 index 0000000000000..ffea8d6173d51 --- /dev/null +++ b/cpp/src/arrow/vendored/datetime/tz.cpp @@ -0,0 +1,3794 @@ +// The MIT License (MIT) +// +// Copyright (c) 2015, 2016, 2017 Howard Hinnant +// Copyright (c) 2015 Ville Voutilainen +// Copyright (c) 2016 Alexander Kormanovsky +// Copyright (c) 2016, 2017 Jiangang Zhuang +// Copyright (c) 2017 Nicolas Veloz Savino +// Copyright (c) 2017 Florian Dang +// Copyright (c) 2017 Aaron Bishop +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Our apologies. When the previous paragraph was written, lowercase had not yet +// been invented (that would involve another several millennia of evolution). +// We did not mean to shout. + +#ifdef _WIN32 + // windows.h will be included directly and indirectly (e.g. by curl). + // We need to define these macros to prevent windows.h bringing in + // more than we need and do it early so windows.h doesn't get included + // without these macros having been defined. + // min/max macros interfere with the C++ versions. +# ifndef NOMINMAX +# define NOMINMAX +# endif + // We don't need all that Windows has to offer. +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif + + // for wcstombs +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif + + // None of this happens with the MS SDK (at least VS14 which I tested), but: + // Compiling with mingw, we get "error: 'KF_FLAG_DEFAULT' was not declared in this scope." + // and error: 'SHGetKnownFolderPath' was not declared in this scope.". + // It seems when using mingw NTDDI_VERSION is undefined and that + // causes KNOWN_FOLDER_FLAG and the KF_ flags to not get defined. + // So we must define NTDDI_VERSION to get those flags on mingw. + // The docs say though here: + // https://msdn.microsoft.com/en-nz/library/windows/desktop/aa383745(v=vs.85).aspx + // that "If you define NTDDI_VERSION, you must also define _WIN32_WINNT." + // So we declare we require Vista or greater. +# ifdef __MINGW32__ + +# ifndef NTDDI_VERSION +# define NTDDI_VERSION 0x06000000 +# define _WIN32_WINNT _WIN32_WINNT_VISTA +# elif NTDDI_VERSION < 0x06000000 +# warning "If this fails to compile NTDDI_VERSION may be to low. See comments above." +# endif + // But once we define the values above we then get this linker error: + // "tz.cpp:(.rdata$.refptr.FOLDERID_Downloads[.refptr.FOLDERID_Downloads]+0x0): " + // "undefined reference to `FOLDERID_Downloads'" + // which #include cures see: + // https://support.microsoft.com/en-us/kb/130869 +# include + // But with included, the error moves on to: + // error: 'FOLDERID_Downloads' was not declared in this scope + // Which #include cures. +# include + +# endif // __MINGW32__ + +# include +#endif // _WIN32 + +#include "tz_private.h" + +#ifdef __APPLE__ +# include "ios.h" +#else +# define TARGET_OS_IPHONE 0 +#endif + +#if USE_OS_TZDB +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if USE_OS_TZDB +# include +#endif +#include +#include +#include +#include +#include + +// unistd.h is used on some platforms as part of the the means to get +// the current time zone. On Win32 windows.h provides a means to do it. +// gcc/mingw supports unistd.h on Win32 but MSVC does not. + +#ifdef _WIN32 +# include // _unlink etc. + +# if defined(__clang__) + struct IUnknown; // fix for issue with static_cast<> in objbase.h + // (see https://github.com/philsquared/Catch/issues/690) +# endif + +# include // CoTaskFree, ShGetKnownFolderPath etc. +# if HAS_REMOTE_API +# include // _mkdir +# include // ShFileOperation etc. +# endif // HAS_REMOTE_API +#else // !_WIN32 +# include +# if !USE_OS_TZDB +# include +# endif +# include +# include +# if !USE_SHELL_API +# include +# include +# include +# include +# include +# include +# endif //!USE_SHELL_API +#endif // !_WIN32 + + +#if HAS_REMOTE_API + // Note curl includes windows.h so we must include curl AFTER definitions of things + // that affect windows.h such as NOMINMAX. +#if defined(_MSC_VER) && defined(SHORTENED_CURL_INCLUDE) + // For rmt_curl nuget package +# include +#else +# include +#endif +#endif + +#ifdef _WIN32 +static CONSTDATA char folder_delimiter = '\\'; +#else // !_WIN32 +static CONSTDATA char folder_delimiter = '/'; +#endif // !_WIN32 + +#if defined(__GNUC__) && __GNUC__ < 5 + // GCC 4.9 Bug 61489 Wrong warning with -Wmissing-field-initializers +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif // defined(__GNUC__) && __GNUC__ < 5 + +#if !USE_OS_TZDB + +# ifdef _WIN32 + +namespace +{ + struct task_mem_deleter + { + void operator()(wchar_t buf[]) + { + if (buf != nullptr) + CoTaskMemFree(buf); + } + }; + using co_task_mem_ptr = std::unique_ptr; +} + +// We might need to know certain locations even if not using the remote API, +// so keep these routines out of that block for now. +static +std::string +get_known_folder(const GUID& folderid) +{ + std::string folder; + PWSTR pfolder = nullptr; + HRESULT hr = SHGetKnownFolderPath(folderid, KF_FLAG_DEFAULT, nullptr, &pfolder); + if (SUCCEEDED(hr)) + { + co_task_mem_ptr folder_ptr(pfolder); + folder = std::string(folder_ptr.get(), folder_ptr.get() + wcslen(folder_ptr.get())); + } + return folder; +} + +// Usually something like "c:\Users\username\Downloads". +static +std::string +get_download_folder() +{ + return get_known_folder(FOLDERID_Downloads); +} + +# else // !_WIN32 + +# if !defined(INSTALL) || HAS_REMOTE_API + +static +std::string +expand_path(std::string path) +{ +# if TARGET_OS_IPHONE + return date::iOSUtils::get_tzdata_path(); +# else // !TARGET_OS_IPHONE + ::wordexp_t w{}; + std::unique_ptr<::wordexp_t, void(*)(::wordexp_t*)> hold{&w, ::wordfree}; + ::wordexp(path.c_str(), &w, 0); + if (w.we_wordc != 1) + throw std::runtime_error("Cannot expand path: " + path); + path = w.we_wordv[0]; + return path; +# endif // !TARGET_OS_IPHONE +} + +static +std::string +get_download_folder() +{ + return expand_path("~/Downloads"); +} + +# endif // !defined(INSTALL) || HAS_REMOTE_API + +# endif // !_WIN32 + +#endif // !USE_OS_TZDB + +namespace arrow +{ +namespace util +{ +namespace date +{ +// +---------------------+ +// | Begin Configuration | +// +---------------------+ + +using namespace detail; + +#if !USE_OS_TZDB + +static +std::string& +access_install() +{ + static std::string install +#ifndef INSTALL + + = get_download_folder() + folder_delimiter + "tzdata"; + +#else // !INSTALL + +# define STRINGIZEIMP(x) #x +# define STRINGIZE(x) STRINGIZEIMP(x) + + = STRINGIZE(INSTALL) + std::string(1, folder_delimiter) + "tzdata"; + + #undef STRINGIZEIMP + #undef STRINGIZE +#endif // !INSTALL + + return install; +} + +void +set_install(const std::string& s) +{ + access_install() = s; +} + +static +const std::string& +get_install() +{ + static const std::string& ref = access_install(); + return ref; +} + +#if HAS_REMOTE_API +static +std::string +get_download_gz_file(const std::string& version) +{ + auto file = get_install() + version + ".tar.gz"; + return file; +} +#endif // HAS_REMOTE_API + +#endif // !USE_OS_TZDB + +// These can be used to reduce the range of the database to save memory +CONSTDATA auto min_year = date::year::min(); +CONSTDATA auto max_year = date::year::max(); + +CONSTDATA auto min_day = date::jan/1; +CONSTDATA auto max_day = date::dec/31; + +#if USE_OS_TZDB + +CONSTCD14 const sys_seconds min_seconds = sys_days(min_year/min_day); + +#endif // USE_OS_TZDB + +#ifndef _WIN32 + +static +std::string +discover_tz_dir() +{ + struct stat sb; + using namespace std; +# ifndef __APPLE__ + CONSTDATA auto tz_dir_default = "/usr/share/zoneinfo"; + CONSTDATA auto tz_dir_buildroot = "/usr/share/zoneinfo/uclibc"; + + // Check special path which is valid for buildroot with uclibc builds + if(stat(tz_dir_buildroot, &sb) == 0 && S_ISDIR(sb.st_mode)) + return tz_dir_buildroot; + else if(stat(tz_dir_default, &sb) == 0 && S_ISDIR(sb.st_mode)) + return tz_dir_default; + else + throw runtime_error("discover_tz_dir failed to find zoneinfo\n"); +# else // __APPLE__ +# if TARGET_OS_IPHONE + return "/var/db/timezone/zoneinfo"; +# else + CONSTDATA auto timezone = "/etc/localtime"; + if (!(lstat(timezone, &sb) == 0 && S_ISLNK(sb.st_mode) && sb.st_size > 0)) + throw runtime_error("discover_tz_dir failed\n"); + string result; + char rp[PATH_MAX+1] = {}; + if (readlink(timezone, rp, sizeof(rp)-1) > 0) + result = string(rp); + else + throw system_error(errno, system_category(), "readlink() failed"); + auto i = result.find("zoneinfo"); + if (i == string::npos) + throw runtime_error("discover_tz_dir failed to find zoneinfo\n"); + i = result.find('/', i); + if (i == string::npos) + throw runtime_error("discover_tz_dir failed to find '/'\n"); + return result.substr(0, i); +# endif +# endif // __APPLE__ +} + +static +const std::string& +get_tz_dir() +{ + static const std::string tz_dir = discover_tz_dir(); + return tz_dir; +} + +#endif + +// +-------------------+ +// | End Configuration | +// +-------------------+ + +namespace detail +{ +struct undocumented {explicit undocumented() = default;}; +} + +#ifndef _MSC_VER +static_assert(min_year <= max_year, "Configuration error"); +#endif + +static std::unique_ptr init_tzdb(); + +tzdb_list::~tzdb_list() +{ + const tzdb* ptr = head_; + head_ = nullptr; + while (ptr != nullptr) + { + auto next = ptr->next; + delete ptr; + ptr = next; + } +} + +tzdb_list::tzdb_list(tzdb_list&& x) noexcept + : head_{x.head_.exchange(nullptr)} +{ +} + +void +tzdb_list::push_front(tzdb* tzdb) noexcept +{ + tzdb->next = head_; + head_ = tzdb; +} + +tzdb_list::const_iterator +tzdb_list::erase_after(const_iterator p) noexcept +{ + auto t = p.p_->next; + p.p_->next = p.p_->next->next; + delete t; + return ++p; +} + +struct tzdb_list::undocumented_helper +{ + static void push_front(tzdb_list& db_list, tzdb* tzdb) noexcept + { + db_list.push_front(tzdb); + } +}; + +static +tzdb_list +create_tzdb() +{ + tzdb_list tz_db; + tzdb_list::undocumented_helper::push_front(tz_db, init_tzdb().release()); + return tz_db; +} + +tzdb_list& +get_tzdb_list() +{ + static tzdb_list tz_db = create_tzdb(); + return tz_db; +} + +#if !USE_OS_TZDB + +#ifdef _WIN32 + +static +void +sort_zone_mappings(std::vector& mappings) +{ + std::sort(mappings.begin(), mappings.end(), + [](const date::detail::timezone_mapping& lhs, + const date::detail::timezone_mapping& rhs)->bool + { + auto other_result = lhs.other.compare(rhs.other); + if (other_result < 0) + return true; + else if (other_result == 0) + { + auto territory_result = lhs.territory.compare(rhs.territory); + if (territory_result < 0) + return true; + else if (territory_result == 0) + { + if (lhs.type < rhs.type) + return true; + } + } + return false; + }); +} + +static +bool +native_to_standard_timezone_name(const std::string& native_tz_name, + std::string& standard_tz_name) +{ + // TOOD! Need be a case insensitive compare? + if (native_tz_name == "UTC") + { + standard_tz_name = "Etc/UTC"; + return true; + } + standard_tz_name.clear(); + // TODO! we can improve on linear search. + const auto& mappings = date::get_tzdb().mappings; + for (const auto& tzm : mappings) + { + if (tzm.other == native_tz_name) + { + standard_tz_name = tzm.type; + return true; + } + } + return false; +} + +// Parse this XML file: +// http://unicode.org/repos/cldr/trunk/common/supplemental/windowsZones.xml +// The parsing method is designed to be simple and quick. It is not overly +// forgiving of change but it should diagnose basic format issues. +// See timezone_mapping structure for more info. +static +std::vector +load_timezone_mappings_from_xml_file(const std::string& input_path) +{ + std::size_t line_num = 0; + std::vector mappings; + std::string line; + + std::ifstream is(input_path); + if (!is.is_open()) + { + // We don't emit file exceptions because that's an implementation detail. + std::string msg = "Error opening time zone mapping file \""; + msg += input_path; + msg += "\"."; + throw std::runtime_error(msg); + } + + auto error = [&input_path, &line_num](const char* info) + { + std::string msg = "Error loading time zone mapping file \""; + msg += input_path; + msg += "\" at line "; + msg += std::to_string(line_num); + msg += ": "; + msg += info; + throw std::runtime_error(msg); + }; + // [optional space]a="b" + auto read_attribute = [&line, &error] + (const char* name, std::string& value, std::size_t startPos) + ->std::size_t + { + value.clear(); + // Skip leading space before attribute name. + std::size_t spos = line.find_first_not_of(' ', startPos); + if (spos == std::string::npos) + spos = startPos; + // Assume everything up to next = is the attribute name + // and that an = will always delimit that. + std::size_t epos = line.find('=', spos); + if (epos == std::string::npos) + error("Expected \'=\' right after attribute name."); + std::size_t name_len = epos - spos; + // Expect the name we find matches the name we expect. + if (line.compare(spos, name_len, name) != 0) + { + std::string msg; + msg = "Expected attribute name \'"; + msg += name; + msg += "\' around position "; + msg += std::to_string(spos); + msg += " but found something else."; + error(msg.c_str()); + } + ++epos; // Skip the '=' that is after the attribute name. + spos = epos; + if (spos < line.length() && line[spos] == '\"') + ++spos; // Skip the quote that is before the attribute value. + else + { + std::string msg = "Expected '\"' to begin value of attribute \'"; + msg += name; + msg += "\'."; + error(msg.c_str()); + } + epos = line.find('\"', spos); + if (epos == std::string::npos) + { + std::string msg = "Expected '\"' to end value of attribute \'"; + msg += name; + msg += "\'."; + error(msg.c_str()); + } + // Extract everything in between the quotes. Note no escaping is done. + std::size_t value_len = epos - spos; + value.assign(line, spos, value_len); + ++epos; // Skip the quote that is after the attribute value; + return epos; + }; + + // Quick but not overly forgiving XML mapping file processing. + bool mapTimezonesOpenTagFound = false; + bool mapTimezonesCloseTagFound = false; + std::size_t mapZonePos = std::string::npos; + std::size_t mapTimezonesPos = std::string::npos; + CONSTDATA char mapTimeZonesOpeningTag[] = { ""); + mapTimezonesCloseTagFound = (mapTimezonesPos != std::string::npos); + if (!mapTimezonesCloseTagFound) + { + std::size_t commentPos = line.find(" " << x.target_; +} + +// leap + +leap::leap(const std::string& s, detail::undocumented) +{ + using namespace date; + std::istringstream in(s); + in.exceptions(std::ios::failbit | std::ios::badbit); + std::string word; + int y; + MonthDayTime date; + in >> word >> y >> date; + date_ = date.to_time_point(year(y)); +} + +static +bool +file_exists(const std::string& filename) +{ +#ifdef _WIN32 + return ::_access(filename.c_str(), 0) == 0; +#else + return ::access(filename.c_str(), F_OK) == 0; +#endif +} + +#if HAS_REMOTE_API + +// CURL tools + +static +int +curl_global() +{ + if (::curl_global_init(CURL_GLOBAL_DEFAULT) != 0) + throw std::runtime_error("CURL global initialization failed"); + return 0; +} + +namespace +{ + +struct curl_deleter +{ + void operator()(CURL* p) const + { + ::curl_easy_cleanup(p); + } +}; + +} // unnamed namespace + +static +std::unique_ptr +curl_init() +{ + static const auto curl_is_now_initiailized = curl_global(); + (void)curl_is_now_initiailized; + return std::unique_ptr{::curl_easy_init()}; +} + +static +bool +download_to_string(const std::string& url, std::string& str) +{ + str.clear(); + auto curl = curl_init(); + if (!curl) + return false; + std::string version; + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_write_callback write_cb = [](char* contents, std::size_t size, std::size_t nmemb, + void* userp) -> std::size_t + { + auto& userstr = *static_cast(userp); + auto realsize = size * nmemb; + userstr.append(contents, realsize); + return realsize; + }; + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, write_cb); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &str); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, false); + auto res = curl_easy_perform(curl.get()); + return (res == CURLE_OK); +} + +namespace +{ + enum class download_file_options { binary, text }; +} + +static +bool +download_to_file(const std::string& url, const std::string& local_filename, + download_file_options opts) +{ + auto curl = curl_init(); + if (!curl) + return false; + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, false); + curl_write_callback write_cb = [](char* contents, std::size_t size, std::size_t nmemb, + void* userp) -> std::size_t + { + auto& of = *static_cast(userp); + auto realsize = size * nmemb; + of.write(contents, static_cast(realsize)); + return realsize; + }; + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, write_cb); + decltype(curl_easy_perform(curl.get())) res; + { + std::ofstream of(local_filename, + opts == download_file_options::binary ? + std::ofstream::out | std::ofstream::binary : + std::ofstream::out); + of.exceptions(std::ios::badbit); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &of); + res = curl_easy_perform(curl.get()); + } + return res == CURLE_OK; +} + +std::string +remote_version() +{ + std::string version; + std::string str; + if (download_to_string("https://www.iana.org/time-zones", str)) + { + CONSTDATA char db[] = "/time-zones/releases/tzdata"; + CONSTDATA auto db_size = sizeof(db) - 1; + auto p = str.find(db, 0, db_size); + const int ver_str_len = 5; + if (p != std::string::npos && p + (db_size + ver_str_len) <= str.size()) + version = str.substr(p + db_size, ver_str_len); + } + return version; +} + + +// TODO! Using system() create a process and a console window. +// This is useful to see what errors may occur but is slow and distracting. +// Consider implementing this functionality more directly, such as +// using _mkdir and CreateProcess etc. +// But use the current means now as matches Unix implementations and while +// in proof of concept / testing phase. +// TODO! Use eventually. +static +bool +remove_folder_and_subfolders(const std::string& folder) +{ +# ifdef _WIN32 +# if USE_SHELL_API + // Delete the folder contents by deleting the folder. + std::string cmd = "rd /s /q \""; + cmd += folder; + cmd += '\"'; + return std::system(cmd.c_str()) == EXIT_SUCCESS; +# else // !USE_SHELL_API + // Create a buffer containing the path to delete. It must be terminated + // by two nuls. Who designs these API's... + std::vector from; + from.assign(folder.begin(), folder.end()); + from.push_back('\0'); + from.push_back('\0'); + SHFILEOPSTRUCT fo{}; // Zero initialize. + fo.wFunc = FO_DELETE; + fo.pFrom = from.data(); + fo.fFlags = FOF_NO_UI; + int ret = SHFileOperation(&fo); + if (ret == 0 && !fo.fAnyOperationsAborted) + return true; + return false; +# endif // !USE_SHELL_API +# else // !_WIN32 +# if USE_SHELL_API + return std::system(("rm -R " + folder).c_str()) == EXIT_SUCCESS; +# else // !USE_SHELL_API + struct dir_deleter { + dir_deleter() {} + void operator()(DIR* d) const + { + if (d != nullptr) + { + int result = closedir(d); + assert(result == 0); + } + } + }; + using closedir_ptr = std::unique_ptr; + + std::string filename; + struct stat statbuf; + std::size_t folder_len = folder.length(); + struct dirent* p = nullptr; + + closedir_ptr d(opendir(folder.c_str())); + bool r = d.get() != nullptr; + while (r && (p=readdir(d.get())) != nullptr) + { + if (strcmp(p->d_name, ".") == 0 || strcmp(p->d_name, "..") == 0) + continue; + + // + 2 for path delimiter and nul terminator. + std::size_t buf_len = folder_len + strlen(p->d_name) + 2; + filename.resize(buf_len); + std::size_t path_len = static_cast( + snprintf(&filename[0], buf_len, "%s/%s", folder.c_str(), p->d_name)); + assert(path_len == buf_len - 1); + filename.resize(path_len); + + if (stat(filename.c_str(), &statbuf) == 0) + r = S_ISDIR(statbuf.st_mode) + ? remove_folder_and_subfolders(filename) + : unlink(filename.c_str()) == 0; + } + d.reset(); + + if (r) + r = rmdir(folder.c_str()) == 0; + + return r; +# endif // !USE_SHELL_API +# endif // !_WIN32 +} + +static +bool +make_directory(const std::string& folder) +{ +# ifdef _WIN32 +# if USE_SHELL_API + // Re-create the folder. + std::string cmd = "mkdir \""; + cmd += folder; + cmd += '\"'; + return std::system(cmd.c_str()) == EXIT_SUCCESS; +# else // !USE_SHELL_API + return _mkdir(folder.c_str()) == 0; +# endif // !USE_SHELL_API +# else // !_WIN32 +# if USE_SHELL_API + return std::system(("mkdir " + folder).c_str()) == EXIT_SUCCESS; +# else // !USE_SHELL_API + return mkdir(folder.c_str(), 0777) == 0; +# endif // !USE_SHELL_API +# endif // !_WIN32 +} + +static +bool +delete_file(const std::string& file) +{ +# ifdef _WIN32 +# if USE_SHELL_API + std::string cmd = "del \""; + cmd += file; + cmd += '\"'; + return std::system(cmd.c_str()) == 0; +# else // !USE_SHELL_API + return _unlink(file.c_str()) == 0; +# endif // !USE_SHELL_API +# else // !_WIN32 +# if USE_SHELL_API + return std::system(("rm " + file).c_str()) == EXIT_SUCCESS; +# else // !USE_SHELL_API + return unlink(file.c_str()) == 0; +# endif // !USE_SHELL_API +# endif // !_WIN32 +} + +# ifdef _WIN32 + +static +bool +move_file(const std::string& from, const std::string& to) +{ +# if USE_SHELL_API + std::string cmd = "move \""; + cmd += from; + cmd += "\" \""; + cmd += to; + cmd += '\"'; + return std::system(cmd.c_str()) == EXIT_SUCCESS; +# else // !USE_SHELL_API + return !!::MoveFile(from.c_str(), to.c_str()); +# endif // !USE_SHELL_API +} + +// Usually something like "c:\Program Files". +static +std::string +get_program_folder() +{ + return get_known_folder(FOLDERID_ProgramFiles); +} + +// Note folder can and usually does contain spaces. +static +std::string +get_unzip_program() +{ + std::string path; + + // 7-Zip appears to note its location in the registry. + // If that doesn't work, fall through and take a guess, but it will likely be wrong. + HKEY hKey = nullptr; + if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "SOFTWARE\\7-Zip", 0, KEY_READ, &hKey) == ERROR_SUCCESS) + { + char value_buffer[MAX_PATH + 1]; // fyi 260 at time of writing. + // in/out parameter. Documentation say that size is a count of bytes not chars. + DWORD size = sizeof(value_buffer) - sizeof(value_buffer[0]); + DWORD tzi_type = REG_SZ; + // Testing shows Path key value is "C:\Program Files\7-Zip\" i.e. always with trailing \. + bool got_value = (RegQueryValueExA(hKey, "Path", nullptr, &tzi_type, + reinterpret_cast(value_buffer), &size) == ERROR_SUCCESS); + RegCloseKey(hKey); // Close now incase of throw later. + if (got_value) + { + // Function does not guarantee to null terminate. + value_buffer[size / sizeof(value_buffer[0])] = '\0'; + path = value_buffer; + if (!path.empty()) + { + path += "7z.exe"; + return path; + } + } + } + path += get_program_folder(); + path += folder_delimiter; + path += "7-Zip\\7z.exe"; + return path; +} + +# if !USE_SHELL_API +static +int +run_program(const std::string& command) +{ + STARTUPINFO si{}; + si.cb = sizeof(si); + PROCESS_INFORMATION pi{}; + + // Allegedly CreateProcess overwrites the command line. Ugh. + std::string mutable_command(command); + if (CreateProcess(nullptr, &mutable_command[0], + nullptr, nullptr, FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &si, &pi)) + { + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD exit_code; + bool got_exit_code = !!GetExitCodeProcess(pi.hProcess, &exit_code); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + // Not 100% sure about this still active thing is correct, + // but I'm going with it because I *think* WaitForSingleObject might + // return in some cases without INFINITE-ly waiting. + // But why/wouldn't GetExitCodeProcess return false in that case? + if (got_exit_code && exit_code != STILL_ACTIVE) + return static_cast(exit_code); + } + return EXIT_FAILURE; +} +# endif // !USE_SHELL_API + +static +std::string +get_download_tar_file(const std::string& version) +{ + auto file = get_install(); + file += folder_delimiter; + file += "tzdata"; + file += version; + file += ".tar"; + return file; +} + +static +bool +extract_gz_file(const std::string& version, const std::string& gz_file, + const std::string& dest_folder) +{ + auto unzip_prog = get_unzip_program(); + bool unzip_result = false; + // Use the unzip program to extract the tar file from the archive. + + // Aim to create a string like: + // "C:\Program Files\7-Zip\7z.exe" x "C:\Users\SomeUser\Downloads\tzdata2016d.tar.gz" + // -o"C:\Users\SomeUser\Downloads\tzdata" + std::string cmd; + cmd = '\"'; + cmd += unzip_prog; + cmd += "\" x \""; + cmd += gz_file; + cmd += "\" -o\""; + cmd += dest_folder; + cmd += '\"'; + +# if USE_SHELL_API + // When using shelling out with std::system() extra quotes are required around the + // whole command. It's weird but necessary it seems, see: + // http://stackoverflow.com/q/27975969/576911 + + cmd = "\"" + cmd + "\""; + if (std::system(cmd.c_str()) == EXIT_SUCCESS) + unzip_result = true; +# else // !USE_SHELL_API + if (run_program(cmd) == EXIT_SUCCESS) + unzip_result = true; +# endif // !USE_SHELL_API + if (unzip_result) + delete_file(gz_file); + + // Use the unzip program extract the data from the tar file that was + // just extracted from the archive. + auto tar_file = get_download_tar_file(version); + cmd = '\"'; + cmd += unzip_prog; + cmd += "\" x \""; + cmd += tar_file; + cmd += "\" -o\""; + cmd += get_install(); + cmd += '\"'; +# if USE_SHELL_API + cmd = "\"" + cmd + "\""; + if (std::system(cmd.c_str()) == EXIT_SUCCESS) + unzip_result = true; +# else // !USE_SHELL_API + if (run_program(cmd) == EXIT_SUCCESS) + unzip_result = true; +# endif // !USE_SHELL_API + + if (unzip_result) + delete_file(tar_file); + + return unzip_result; +} + +static +std::string +get_download_mapping_file(const std::string& version) +{ + auto file = get_install() + version + "windowsZones.xml"; + return file; +} + +# else // !_WIN32 + +# if !USE_SHELL_API +static +int +run_program(const char* prog, const char*const args[]) +{ + pid_t pid = fork(); + if (pid == -1) // Child failed to start. + return EXIT_FAILURE; + + if (pid != 0) + { + // We are in the parent. Child started. Wait for it. + pid_t ret; + int status; + while ((ret = waitpid(pid, &status, 0)) == -1) + { + if (errno != EINTR) + break; + } + if (ret != -1) + { + if (WIFEXITED(status)) + return WEXITSTATUS(status); + } + printf("Child issues!\n"); + + return EXIT_FAILURE; // Not sure what status of child is. + } + else // We are in the child process. Start the program the parent wants to run. + { + + if (execv(prog, const_cast(args)) == -1) // Does not return. + { + perror("unreachable 0\n"); + _Exit(127); + } + printf("unreachable 2\n"); + } + printf("unreachable 2\n"); + // Unreachable. + assert(false); + exit(EXIT_FAILURE); + return EXIT_FAILURE; +} +# endif // !USE_SHELL_API + +static +bool +extract_gz_file(const std::string&, const std::string& gz_file, const std::string&) +{ +# if USE_SHELL_API + bool unzipped = std::system(("tar -xzf " + gz_file + " -C " + get_install()).c_str()) == EXIT_SUCCESS; +# else // !USE_SHELL_API + const char prog[] = {"/usr/bin/tar"}; + const char*const args[] = + { + prog, "-xzf", gz_file.c_str(), "-C", get_install().c_str(), nullptr + }; + bool unzipped = (run_program(prog, args) == EXIT_SUCCESS); +# endif // !USE_SHELL_API + if (unzipped) + { + delete_file(gz_file); + return true; + } + return false; +} + +# endif // !_WIN32 + +bool +remote_download(const std::string& version) +{ + assert(!version.empty()); + +# ifdef _WIN32 + // Download folder should be always available for Windows +# else // !_WIN32 + // Create download folder if it does not exist on UNIX system + auto download_folder = get_download_folder(); + if (!file_exists(download_folder)) + { + make_directory(download_folder); + } +# endif // _WIN32 + + auto url = "https://data.iana.org/time-zones/releases/tzdata" + version + + ".tar.gz"; + bool result = download_to_file(url, get_download_gz_file(version), + download_file_options::binary); +# ifdef _WIN32 + if (result) + { + auto mapping_file = get_download_mapping_file(version); + result = download_to_file("http://unicode.org/repos/cldr/trunk/common/" + "supplemental/windowsZones.xml", + mapping_file, download_file_options::text); + } +# endif // _WIN32 + return result; +} + +bool +remote_install(const std::string& version) +{ + auto success = false; + assert(!version.empty()); + + std::string install = get_install(); + auto gz_file = get_download_gz_file(version); + if (file_exists(gz_file)) + { + if (file_exists(install)) + remove_folder_and_subfolders(install); + if (make_directory(install)) + { + if (extract_gz_file(version, gz_file, install)) + success = true; +# ifdef _WIN32 + auto mapping_file_source = get_download_mapping_file(version); + auto mapping_file_dest = get_install(); + mapping_file_dest += folder_delimiter; + mapping_file_dest += "windowsZones.xml"; + if (!move_file(mapping_file_source, mapping_file_dest)) + success = false; +# endif // _WIN32 + } + } + return success; +} + +#endif // HAS_REMOTE_API + +static +std::string +get_version(const std::string& path) +{ + std::string version; + std::ifstream infile(path + "version"); + if (infile.is_open()) + { + infile >> version; + if (!infile.fail()) + return version; + } + else + { + infile.open(path + "NEWS"); + while (infile) + { + infile >> version; + if (version == "Release") + { + infile >> version; + return version; + } + } + } + throw std::runtime_error("Unable to get Timezone database version from " + path); +} + +static +std::unique_ptr +init_tzdb() +{ + using namespace date; + const std::string install = get_install(); + const std::string path = install + folder_delimiter; + std::string line; + bool continue_zone = false; + std::unique_ptr db(new tzdb); + +#if AUTO_DOWNLOAD + if (!file_exists(install)) + { + auto rv = remote_version(); + if (!rv.empty() && remote_download(rv)) + { + if (!remote_install(rv)) + { + std::string msg = "Timezone database version \""; + msg += rv; + msg += "\" did not install correctly to \""; + msg += install; + msg += "\""; + throw std::runtime_error(msg); + } + } + if (!file_exists(install)) + { + std::string msg = "Timezone database not found at \""; + msg += install; + msg += "\""; + throw std::runtime_error(msg); + } + db->version = get_version(path); + } + else + { + db->version = get_version(path); + auto rv = remote_version(); + if (!rv.empty() && db->version != rv) + { + if (remote_download(rv)) + { + remote_install(rv); + db->version = get_version(path); + } + } + } +#else // !AUTO_DOWNLOAD + if (!file_exists(install)) + { + std::string msg = "Timezone database not found at \""; + msg += install; + msg += "\""; + throw std::runtime_error(msg); + } + db->version = get_version(path); +#endif // !AUTO_DOWNLOAD + + CONSTDATA char*const files[] = + { + "africa", "antarctica", "asia", "australasia", "backward", "etcetera", "europe", + "pacificnew", "northamerica", "southamerica", "systemv", "leapseconds" + }; + + for (const auto& filename : files) + { + std::ifstream infile(path + filename); + while (infile) + { + std::getline(infile, line); + if (!line.empty() && line[0] != '#') + { + std::istringstream in(line); + std::string word; + in >> word; + if (word == "Rule") + { + db->rules.push_back(Rule(line)); + continue_zone = false; + } + else if (word == "Link") + { + db->links.push_back(link(line)); + continue_zone = false; + } + else if (word == "Leap") + { + db->leaps.push_back(leap(line, detail::undocumented{})); + continue_zone = false; + } + else if (word == "Zone") + { + db->zones.push_back(time_zone(line, detail::undocumented{})); + continue_zone = true; + } + else if (line[0] == '\t' && continue_zone) + { + db->zones.back().add(line); + } + else + { + std::cerr << line << '\n'; + } + } + } + } + std::sort(db->rules.begin(), db->rules.end()); + Rule::split_overlaps(db->rules); + std::sort(db->zones.begin(), db->zones.end()); + db->zones.shrink_to_fit(); + std::sort(db->links.begin(), db->links.end()); + db->links.shrink_to_fit(); + std::sort(db->leaps.begin(), db->leaps.end()); + db->leaps.shrink_to_fit(); + +#ifdef _WIN32 + std::string mapping_file = get_install() + folder_delimiter + "windowsZones.xml"; + db->mappings = load_timezone_mappings_from_xml_file(mapping_file); + sort_zone_mappings(db->mappings); +#endif // _WIN32 + + return db; +} + +const tzdb& +reload_tzdb() +{ +#if AUTO_DOWNLOAD + auto const& v = get_tzdb_list().front().version; + if (!v.empty() && v == remote_version()) + return get_tzdb_list().front(); +#endif // AUTO_DOWNLOAD + tzdb_list::undocumented_helper::push_front(get_tzdb_list(), init_tzdb().release()); + return get_tzdb_list().front(); +} + +#endif // !USE_OS_TZDB + +const tzdb& +get_tzdb() +{ + return get_tzdb_list().front(); +} + +const time_zone* +#if HAS_STRING_VIEW +tzdb::locate_zone(std::string_view tz_name) const +#else +tzdb::locate_zone(const std::string& tz_name) const +#endif +{ + auto zi = std::lower_bound(zones.begin(), zones.end(), tz_name, +#if HAS_STRING_VIEW + [](const time_zone& z, const std::string_view& nm) +#else + [](const time_zone& z, const std::string& nm) +#endif + { + return z.name() < nm; + }); + if (zi == zones.end() || zi->name() != tz_name) + { +#if !USE_OS_TZDB + auto li = std::lower_bound(links.begin(), links.end(), tz_name, +#if HAS_STRING_VIEW + [](const link& z, const std::string_view& nm) +#else + [](const link& z, const std::string& nm) +#endif + { + return z.name() < nm; + }); + if (li != links.end() && li->name() == tz_name) + { + zi = std::lower_bound(zones.begin(), zones.end(), li->target(), + [](const time_zone& z, const std::string& nm) + { + return z.name() < nm; + }); + if (zi != zones.end() && zi->name() == li->target()) + return &*zi; + } +#endif // !USE_OS_TZDB + throw std::runtime_error(std::string(tz_name) + " not found in timezone database"); + } + return &*zi; +} + +const time_zone* +#if HAS_STRING_VIEW +locate_zone(std::string_view tz_name) +#else +locate_zone(const std::string& tz_name) +#endif +{ + return get_tzdb().locate_zone(tz_name); +} + +#if USE_OS_TZDB + +std::ostream& +operator<<(std::ostream& os, const tzdb& db) +{ + os << "Version: " << db.version << "\n\n"; + for (const auto& x : db.zones) + os << x << '\n'; +#if !MISSING_LEAP_SECONDS + os << '\n'; + for (const auto& x : db.leaps) + os << x << '\n'; +#endif // !MISSING_LEAP_SECONDS + return os; +} + +#else // !USE_OS_TZDB + +std::ostream& +operator<<(std::ostream& os, const tzdb& db) +{ + os << "Version: " << db.version << '\n'; + std::string title("--------------------------------------------" + "--------------------------------------------\n" + "Name ""Start Y ""End Y " + "Beginning ""Offset " + "Designator\n" + "--------------------------------------------" + "--------------------------------------------\n"); + int count = 0; + for (const auto& x : db.rules) + { + if (count++ % 50 == 0) + os << title; + os << x << '\n'; + } + os << '\n'; + title = std::string("---------------------------------------------------------" + "--------------------------------------------------------\n" + "Name ""Offset " + "Rule ""Abrev ""Until\n" + "---------------------------------------------------------" + "--------------------------------------------------------\n"); + count = 0; + for (const auto& x : db.zones) + { + if (count++ % 10 == 0) + os << title; + os << x << '\n'; + } + os << '\n'; + title = std::string("---------------------------------------------------------" + "--------------------------------------------------------\n" + "Alias ""To\n" + "---------------------------------------------------------" + "--------------------------------------------------------\n"); + count = 0; + for (const auto& x : db.links) + { + if (count++ % 45 == 0) + os << title; + os << x << '\n'; + } + os << '\n'; + title = std::string("---------------------------------------------------------" + "--------------------------------------------------------\n" + "Leap second on\n" + "---------------------------------------------------------" + "--------------------------------------------------------\n"); + os << title; + for (const auto& x : db.leaps) + os << x << '\n'; + return os; +} + +#endif // !USE_OS_TZDB + +// ----------------------- + +#ifdef _WIN32 + +static +std::string +getTimeZoneKeyName() +{ + DYNAMIC_TIME_ZONE_INFORMATION dtzi{}; + auto result = GetDynamicTimeZoneInformation(&dtzi); + if (result == TIME_ZONE_ID_INVALID) + throw std::runtime_error("current_zone(): GetDynamicTimeZoneInformation()" + " reported TIME_ZONE_ID_INVALID."); + auto wlen = wcslen(dtzi.TimeZoneKeyName); + char buf[128] = {}; + assert(sizeof(buf) >= wlen+1); + wcstombs(buf, dtzi.TimeZoneKeyName, wlen); + if (strcmp(buf, "Coordinated Universal Time") == 0) + return "UTC"; + return buf; +} + +const time_zone* +tzdb::current_zone() const +{ + std::string win_tzid = getTimeZoneKeyName(); + std::string standard_tzid; + if (!native_to_standard_timezone_name(win_tzid, standard_tzid)) + { + std::string msg; + msg = "current_zone() failed: A mapping from the Windows Time Zone id \""; + msg += win_tzid; + msg += "\" was not found in the time zone mapping database."; + throw std::runtime_error(msg); + } + return locate_zone(standard_tzid); +} + +#else // !_WIN32 + +const time_zone* +tzdb::current_zone() const +{ + // On some OS's a file called /etc/localtime may + // exist and it may be either a real file + // containing time zone details or a symlink to such a file. + // On MacOS and BSD Unix if this file is a symlink it + // might resolve to a path like this: + // "/usr/share/zoneinfo/America/Los_Angeles" + // If it does, we try to determine the current + // timezone from the remainder of the path by removing the prefix + // and hoping the rest resolves to a valid timezone. + // It may not always work though. If it doesn't then an + // exception will be thrown by local_timezone. + // The path may also take a relative form: + // "../usr/share/zoneinfo/America/Los_Angeles". + { + struct stat sb; + CONSTDATA auto timezone = "/etc/localtime"; + if (lstat(timezone, &sb) == 0 && S_ISLNK(sb.st_mode) && sb.st_size > 0) { + using namespace std; + string result; + char rp[PATH_MAX+1] = {}; + if (readlink(timezone, rp, sizeof(rp)-1) > 0) + result = string(rp); + else + throw system_error(errno, system_category(), "readlink() failed"); + + const size_t pos = result.find(get_tz_dir()); + if (pos != result.npos) + result.erase(0, get_tz_dir().size() + 1 + pos); + return locate_zone(result); + } + } + // On embedded systems e.g. buildroot with uclibc the timezone is linked + // into /etc/TZ which is a symlink to path like this: + // "/usr/share/zoneinfo/uclibc/America/Los_Angeles" + // If it does, we try to determine the current + // timezone from the remainder of the path by removing the prefix + // and hoping the rest resolves to valid timezone. + // It may not always work though. If it doesn't then an + // exception will be thrown by local_timezone. + // The path may also take a relative form: + // "../usr/share/zoneinfo/uclibc/America/Los_Angeles". + { + struct stat sb; + CONSTDATA auto timezone = "/etc/TZ"; + if (lstat(timezone, &sb) == 0 && S_ISLNK(sb.st_mode) && sb.st_size > 0) { + using namespace std; + string result; + char rp[PATH_MAX+1] = {}; + if (readlink(timezone, rp, sizeof(rp)-1) > 0) + result = string(rp); + else + throw system_error(errno, system_category(), "readlink() failed"); + + const size_t pos = result.find(get_tz_dir()); + if (pos != result.npos) + result.erase(0, get_tz_dir().size() + 1 + pos); + return locate_zone(result); + } + } + { + // On some versions of some linux distro's (e.g. Ubuntu), + // the current timezone might be in the first line of + // the /etc/timezone file. + std::ifstream timezone_file("/etc/timezone"); + if (timezone_file.is_open()) + { + std::string result; + std::getline(timezone_file, result); + if (!result.empty()) + return locate_zone(result); + } + // Fall through to try other means. + } + { + // On some versions of some bsd distro's (e.g. FreeBSD), + // the current timezone might be in the first line of + // the /var/db/zoneinfo file. + std::ifstream timezone_file("/var/db/zoneinfo"); + if (timezone_file.is_open()) + { + std::string result; + std::getline(timezone_file, result); + if (!result.empty()) + return locate_zone(result); + } + // Fall through to try other means. + } + { + // On some versions of some bsd distro's (e.g. iOS), + // it is not possible to use file based approach, + // we switch to system API, calling functions in + // CoreFoundation framework. +#if TARGET_OS_IPHONE + std::string result = date::iOSUtils::get_current_timezone(); + if (!result.empty()) + return locate_zone(result); +#endif + // Fall through to try other means. + } + { + // On some versions of some linux distro's (e.g. Red Hat), + // the current timezone might be in the first line of + // the /etc/sysconfig/clock file as: + // ZONE="US/Eastern" + std::ifstream timezone_file("/etc/sysconfig/clock"); + std::string result; + while (timezone_file) + { + std::getline(timezone_file, result); + auto p = result.find("ZONE=\""); + if (p != std::string::npos) + { + result.erase(p, p+6); + result.erase(result.rfind('"')); + return locate_zone(result); + } + } + // Fall through to try other means. + } + throw std::runtime_error("Could not get current timezone"); +} + +#endif // !_WIN32 + +const time_zone* +current_zone() +{ + return get_tzdb().current_zone(); +} + +} // namespace date +} // namespace util +} // namespace arrow + +#if defined(__GNUC__) && __GNUC__ < 5 +# pragma GCC diagnostic pop +#endif diff --git a/cpp/src/arrow/vendored/datetime/tz.h b/cpp/src/arrow/vendored/datetime/tz.h new file mode 100644 index 0000000000000..db78b2df971d2 --- /dev/null +++ b/cpp/src/arrow/vendored/datetime/tz.h @@ -0,0 +1,2593 @@ +#ifndef TZ_H +#define TZ_H + +// The MIT License (MIT) +// +// Copyright (c) 2015, 2016, 2017 Howard Hinnant +// Copyright (c) 2017 Jiangang Zhuang +// Copyright (c) 2017 Aaron Bishop +// Copyright (c) 2017 Tomasz Kamiński +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Our apologies. When the previous paragraph was written, lowercase had not yet +// been invented (that would involve another several millennia of evolution). +// We did not mean to shout. + +// Get more recent database at http://www.iana.org/time-zones + +// The notion of "current timezone" is something the operating system is expected to "just +// know". How it knows this is system specific. It's often a value set by the user at OS +// installation time and recorded by the OS somewhere. On Linux and Mac systems the current +// timezone name is obtained by looking at the name or contents of a particular file on +// disk. On Windows the current timezone name comes from the registry. In either method, +// there is no guarantee that the "native" current timezone name obtained will match any +// of the "Standard" names in this library's "database". On Linux, the names usually do +// seem to match so mapping functions to map from native to "Standard" are typically not +// required. On Windows, the names are never "Standard" so mapping is always required. +// Technically any OS may use the mapping process but currently only Windows does use it. + +/////////////////////////////////////////////////// + +// Windows does not support OS timezone database +#ifdef _WIN32 +# define USE_OS_TZDB 0 +#else +# define USE_OS_TZDB 1 +#endif +#define HAS_REMOTE_API 0 + +//////////////////////////////////////////////////// + +#ifndef USE_OS_TZDB +# define USE_OS_TZDB 0 +#endif + +#ifndef HAS_REMOTE_API +# if USE_OS_TZDB == 0 +# ifdef _WIN32 +# define HAS_REMOTE_API 0 +# else +# define HAS_REMOTE_API 1 +# endif +# else // HAS_REMOTE_API makes no since when using the OS timezone database +# define HAS_REMOTE_API 0 +# endif +#endif + +#ifdef __clang__ +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wconstant-logical-operand" +#endif + +static_assert(!(USE_OS_TZDB && HAS_REMOTE_API), + "USE_OS_TZDB and HAS_REMOTE_API can not be used together"); + +#ifdef __clang__ +# pragma clang diagnostic pop +#endif + +#ifndef AUTO_DOWNLOAD +# define AUTO_DOWNLOAD HAS_REMOTE_API +#endif + +static_assert(HAS_REMOTE_API == 0 ? AUTO_DOWNLOAD == 0 : true, + "AUTO_DOWNLOAD can not be turned on without HAS_REMOTE_API"); + +#ifndef USE_SHELL_API +# define USE_SHELL_API 1 +#endif + +#if USE_OS_TZDB +# ifdef _WIN32 +# error "USE_OS_TZDB can not be used on Windows" +# endif +# ifndef MISSING_LEAP_SECONDS +# ifdef __APPLE__ +# define MISSING_LEAP_SECONDS 1 +# else +# define MISSING_LEAP_SECONDS 0 +# endif +# endif +#else +# define MISSING_LEAP_SECONDS 0 +#endif + +#ifndef HAS_DEDUCTION_GUIDES +# if __cplusplus >= 201703 +# define HAS_DEDUCTION_GUIDES 1 +# else +# define HAS_DEDUCTION_GUIDES 0 +# endif +#endif // HAS_DEDUCTION_GUIDES + +#include "date.h" + +#if defined(_MSC_VER) && (_MSC_VER < 1900) +#include "tz_private.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# ifdef DATE_BUILD_DLL +# define DATE_API __declspec(dllexport) +# elif defined(DATE_USE_DLL) +# define DATE_API __declspec(dllimport) +# else +# define DATE_API +# endif +#else +# ifdef DATE_BUILD_DLL +# define DATE_API __attribute__ ((visibility ("default"))) +# else +# define DATE_API +# endif +#endif + +namespace arrow +{ +namespace util +{ +namespace date +{ + +enum class choose {earliest, latest}; + +namespace detail +{ + struct undocumented; +} + +struct sys_info +{ + sys_seconds begin; + sys_seconds end; + std::chrono::seconds offset; + std::chrono::minutes save; + std::string abbrev; +}; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const sys_info& r) +{ + os << r.begin << '\n'; + os << r.end << '\n'; + os << make_time(r.offset) << "\n"; + os << make_time(r.save) << "\n"; + os << r.abbrev << '\n'; + return os; +} + +struct local_info +{ + enum {unique, nonexistent, ambiguous} result; + sys_info first; + sys_info second; +}; + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const local_info& r) +{ + if (r.result == local_info::nonexistent) + os << "nonexistent between\n"; + else if (r.result == local_info::ambiguous) + os << "ambiguous between\n"; + os << r.first; + if (r.result != local_info::unique) + { + os << "and\n"; + os << r.second; + } + return os; +} + +class nonexistent_local_time + : public std::runtime_error +{ +public: + template + nonexistent_local_time(local_time tp, const local_info& i); + +private: + template + static + std::string + make_msg(local_time tp, const local_info& i); +}; + +template +inline +nonexistent_local_time::nonexistent_local_time(local_time tp, + const local_info& i) + : std::runtime_error(make_msg(tp, i)) +{ +} + +template +std::string +nonexistent_local_time::make_msg(local_time tp, const local_info& i) +{ + assert(i.result == local_info::nonexistent); + std::ostringstream os; + os << tp << " is in a gap between\n" + << local_seconds{i.first.end.time_since_epoch()} + i.first.offset << ' ' + << i.first.abbrev << " and\n" + << local_seconds{i.second.begin.time_since_epoch()} + i.second.offset << ' ' + << i.second.abbrev + << " which are both equivalent to\n" + << i.first.end << " UTC"; + return os.str(); +} + +class ambiguous_local_time + : public std::runtime_error +{ +public: + template + ambiguous_local_time(local_time tp, const local_info& i); + +private: + template + static + std::string + make_msg(local_time tp, const local_info& i); +}; + +template +inline +ambiguous_local_time::ambiguous_local_time(local_time tp, const local_info& i) + : std::runtime_error(make_msg(tp, i)) +{ +} + +template +std::string +ambiguous_local_time::make_msg(local_time tp, const local_info& i) +{ + assert(i.result == local_info::ambiguous); + std::ostringstream os; + os << tp << " is ambiguous. It could be\n" + << tp << ' ' << i.first.abbrev << " == " + << tp - i.first.offset << " UTC or\n" + << tp << ' ' << i.second.abbrev << " == " + << tp - i.second.offset << " UTC"; + return os.str(); +} + +class time_zone; + +#if HAS_STRING_VIEW +DATE_API const time_zone* locate_zone(std::string_view tz_name); +#else +DATE_API const time_zone* locate_zone(const std::string& tz_name); +#endif + +DATE_API const time_zone* current_zone(); + +template +struct zoned_traits +{ +}; + +template <> +struct zoned_traits +{ + static + const time_zone* + default_zone() + { + return date::locate_zone("Etc/UTC"); + } + +#if HAS_STRING_VIEW + + static + const time_zone* + locate_zone(std::string_view name) + { + return date::locate_zone(name); + } + +#else // !HAS_STRING_VIEW + + static + const time_zone* + locate_zone(const std::string& name) + { + return date::locate_zone(name); + } + + static + const time_zone* + locate_zone(const char* name) + { + return date::locate_zone(name); + } + +#endif // !HAS_STRING_VIEW +}; + +template +class zoned_time; + +template +bool +operator==(const zoned_time& x, + const zoned_time& y); + +template +class zoned_time +{ +public: + using duration = typename std::common_type::type; + +private: + TimeZonePtr zone_; + sys_time tp_; + +public: +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::default_zone())> +#endif + zoned_time(); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::default_zone())> +#endif + zoned_time(const sys_time& st); + explicit zoned_time(TimeZonePtr z); + +#if HAS_STRING_VIEW + template ::locate_zone(std::string_view())) + >::value + >::type> + explicit zoned_time(std::string_view name); +#else +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())) + >::value + >::type> +#endif + explicit zoned_time(const std::string& name); +#endif + + template , + sys_time>::value + >::type> + zoned_time(const zoned_time& zt) NOEXCEPT; + + zoned_time(TimeZonePtr z, const sys_time& st); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ()->to_sys(local_time{})), + sys_time + >::value + >::type> +#endif + zoned_time(TimeZonePtr z, const local_time& tp); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ()->to_sys(local_time{}, + choose::earliest)), + sys_time + >::value + >::type> +#endif + zoned_time(TimeZonePtr z, const local_time& tp, choose c); + + template , + sys_time>::value + >::type> + zoned_time(TimeZonePtr z, const zoned_time& zt); + + template , + sys_time>::value + >::type> + zoned_time(TimeZonePtr z, const zoned_time& zt, choose); + +#if HAS_STRING_VIEW + + template ::locate_zone(std::string_view())), + sys_time + >::value + >::type> + zoned_time(std::string_view name, const sys_time& st); + + template ::locate_zone(std::string_view())), + local_time + >::value + >::type> + zoned_time(std::string_view name, const local_time& tp); + + template ::locate_zone(std::string_view())), + local_time, + choose + >::value + >::type> + zoned_time(std::string_view name, const local_time& tp, choose c); + + template ::locate_zone(std::string_view())), + zoned_time + >::value + >::type> + zoned_time(std::string_view name, const zoned_time& zt); + + template ::locate_zone(std::string_view())), + zoned_time, + choose + >::value + >::type> + zoned_time(std::string_view name, const zoned_time& zt, choose); + +#else // !HAS_STRING_VIEW + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + sys_time + >::value + >::type> +#endif + zoned_time(const std::string& name, const sys_time& st); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + sys_time + >::value + >::type> +#endif + zoned_time(const char* name, const sys_time& st); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + local_time + >::value + >::type> +#endif + zoned_time(const std::string& name, const local_time& tp); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + local_time + >::value + >::type> +#endif + zoned_time(const char* name, const local_time& tp); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + local_time, + choose + >::value + >::type> +#endif + zoned_time(const std::string& name, const local_time& tp, choose c); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + local_time, + choose + >::value + >::type> +#endif + zoned_time(const char* name, const local_time& tp, choose c); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + zoned_time + >::value + >::type> +#endif + zoned_time(const std::string& name, const zoned_time& zt); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + zoned_time + >::value + >::type> +#endif + zoned_time(const char* name, const zoned_time& zt); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + zoned_time, + choose + >::value + >::type> +#endif + zoned_time(const std::string& name, const zoned_time& zt, choose); + +#if !defined(_MSC_VER) || (_MSC_VER > 1900) + template ::locate_zone(std::string())), + zoned_time, + choose + >::value + >::type> +#endif + zoned_time(const char* name, const zoned_time& zt, choose); + +#endif // !HAS_STRING_VIEW + + zoned_time& operator=(const sys_time& st); + zoned_time& operator=(const local_time& ut); + + explicit operator sys_time() const; + explicit operator local_time() const; + + TimeZonePtr get_time_zone() const; + local_time get_local_time() const; + sys_time get_sys_time() const; + sys_info get_info() const; + + template + friend + bool + operator==(const zoned_time& x, + const zoned_time& y); + + template + friend + std::basic_ostream& + operator<<(std::basic_ostream& os, + const zoned_time& t); + +private: + template friend class zoned_time; +}; + +using zoned_seconds = zoned_time; + +#if HAS_DEDUCTION_GUIDES + +zoned_time() + -> zoned_time; + +template +zoned_time(sys_time) + -> zoned_time>; + +template +zoned_time(TimeZonePtr) + -> zoned_time; + +template +zoned_time(TimeZonePtr, sys_time) + -> zoned_time, TimeZonePtr>; + +template +zoned_time(TimeZonePtr, local_time, choose = choose::earliest) + -> zoned_time, TimeZonePtr>; + +#if HAS_STRING_VIEW + +zoned_time(std::string_view) + -> zoned_time; + +template +zoned_time(std::string_view, sys_time) + -> zoned_time>; + +template +zoned_time(std::string_view, local_time, choose = choose::earliest) + -> zoned_time>; + +#else // !HAS_STRING_VIEW + +zoned_time(std::string) + -> zoned_time; + +template +zoned_time(std::string, sys_time) + -> zoned_time>; + +template +zoned_time(std::string, local_time, choose = choose::earliest) + -> zoned_time>; + +#endif // !HAS_STRING_VIEW + +template +zoned_time(const char*, sys_time) + -> zoned_time>; + +template +zoned_time(const char*, local_time, choose = choose::earliest) + -> zoned_time>; + +template +zoned_time(TimeZonePtr, zoned_time, choose = choose::earliest) + -> zoned_time; + +#endif // HAS_DEDUCTION_GUIDES + +template +inline +bool +operator==(const zoned_time& x, + const zoned_time& y) +{ + return x.zone_ == y.zone_ && x.tp_ == y.tp_; +} + +template +inline +bool +operator!=(const zoned_time& x, + const zoned_time& y) +{ + return !(x == y); +} + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) + +namespace detail +{ +# if USE_OS_TZDB + struct transition; + struct expanded_ttinfo; +# else // !USE_OS_TZDB + struct zonelet; + class Rule; +# endif // !USE_OS_TZDB +} + +#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) + +class time_zone +{ +private: + std::string name_; +#if USE_OS_TZDB + std::vector transitions_; + std::vector ttinfos_; +#else // !USE_OS_TZDB + std::vector zonelets_; +#endif // !USE_OS_TZDB + std::unique_ptr adjusted_; + +public: +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) + time_zone(time_zone&&) = default; + time_zone& operator=(time_zone&&) = default; +#else // defined(_MSC_VER) && (_MSC_VER < 1900) + time_zone(time_zone&& src); + time_zone& operator=(time_zone&& src); +#endif // defined(_MSC_VER) && (_MSC_VER < 1900) + + DATE_API explicit time_zone(const std::string& s, detail::undocumented); + + const std::string& name() const NOEXCEPT; + + template sys_info get_info(sys_time st) const; + template local_info get_info(local_time tp) const; + + template + sys_time::type> + to_sys(local_time tp) const; + + template + sys_time::type> + to_sys(local_time tp, choose z) const; + + template + local_time::type> + to_local(sys_time tp) const; + + friend bool operator==(const time_zone& x, const time_zone& y) NOEXCEPT; + friend bool operator< (const time_zone& x, const time_zone& y) NOEXCEPT; + friend DATE_API std::ostream& operator<<(std::ostream& os, const time_zone& z); + +#if !USE_OS_TZDB + DATE_API void add(const std::string& s); +#endif // !USE_OS_TZDB + +private: + DATE_API sys_info get_info_impl(sys_seconds tp) const; + DATE_API local_info get_info_impl(local_seconds tp) const; + + template + sys_time::type> + to_sys_impl(local_time tp, choose z, std::false_type) const; + template + sys_time::type> + to_sys_impl(local_time tp, choose, std::true_type) const; + +#if USE_OS_TZDB + DATE_API void init() const; + DATE_API void init_impl(); + DATE_API sys_info + load_sys_info(std::vector::const_iterator i) const; + + template + DATE_API void + load_data(std::istream& inf, std::int32_t tzh_leapcnt, std::int32_t tzh_timecnt, + std::int32_t tzh_typecnt, std::int32_t tzh_charcnt); +#else // !USE_OS_TZDB + DATE_API sys_info get_info_impl(sys_seconds tp, int timezone) const; + DATE_API void adjust_infos(const std::vector& rules); + DATE_API void parse_info(std::istream& in); +#endif // !USE_OS_TZDB +}; + +#if defined(_MSC_VER) && (_MSC_VER < 1900) + +inline +time_zone::time_zone(time_zone&& src) + : name_(std::move(src.name_)) + , zonelets_(std::move(src.zonelets_)) + , adjusted_(std::move(src.adjusted_)) + {} + +inline +time_zone& +time_zone::operator=(time_zone&& src) +{ + name_ = std::move(src.name_); + zonelets_ = std::move(src.zonelets_); + adjusted_ = std::move(src.adjusted_); + return *this; +} + +#endif // defined(_MSC_VER) && (_MSC_VER < 1900) + +inline +const std::string& +time_zone::name() const NOEXCEPT +{ + return name_; +} + +template +inline +sys_info +time_zone::get_info(sys_time st) const +{ + using namespace std::chrono; + return get_info_impl(date::floor(st)); +} + +template +inline +local_info +time_zone::get_info(local_time tp) const +{ + using namespace std::chrono; + return get_info_impl(date::floor(tp)); +} + +template +inline +sys_time::type> +time_zone::to_sys(local_time tp) const +{ + return to_sys_impl(tp, choose{}, std::true_type{}); +} + +template +inline +sys_time::type> +time_zone::to_sys(local_time tp, choose z) const +{ + return to_sys_impl(tp, z, std::false_type{}); +} + +template +inline +local_time::type> +time_zone::to_local(sys_time tp) const +{ + using LT = local_time::type>; + auto i = get_info(tp); + return LT{(tp + i.offset).time_since_epoch()}; +} + +inline bool operator==(const time_zone& x, const time_zone& y) NOEXCEPT {return x.name_ == y.name_;} +inline bool operator< (const time_zone& x, const time_zone& y) NOEXCEPT {return x.name_ < y.name_;} + +inline bool operator!=(const time_zone& x, const time_zone& y) NOEXCEPT {return !(x == y);} +inline bool operator> (const time_zone& x, const time_zone& y) NOEXCEPT {return y < x;} +inline bool operator<=(const time_zone& x, const time_zone& y) NOEXCEPT {return !(y < x);} +inline bool operator>=(const time_zone& x, const time_zone& y) NOEXCEPT {return !(x < y);} + +template +sys_time::type> +time_zone::to_sys_impl(local_time tp, choose z, std::false_type) const +{ + using namespace date; + using namespace std::chrono; + auto i = get_info(tp); + if (i.result == local_info::nonexistent) + { + return i.first.end; + } + else if (i.result == local_info::ambiguous) + { + if (z == choose::latest) + return sys_time{tp.time_since_epoch()} - i.second.offset; + } + return sys_time{tp.time_since_epoch()} - i.first.offset; +} + +template +sys_time::type> +time_zone::to_sys_impl(local_time tp, choose, std::true_type) const +{ + using namespace date; + using namespace std::chrono; + auto i = get_info(tp); + if (i.result == local_info::nonexistent) + throw nonexistent_local_time(tp, i); + else if (i.result == local_info::ambiguous) + throw ambiguous_local_time(tp, i); + return sys_time{tp.time_since_epoch()} - i.first.offset; +} + +#if !USE_OS_TZDB + +class link +{ +private: + std::string name_; + std::string target_; +public: + DATE_API explicit link(const std::string& s); + + const std::string& name() const {return name_;} + const std::string& target() const {return target_;} + + friend bool operator==(const link& x, const link& y) {return x.name_ == y.name_;} + friend bool operator< (const link& x, const link& y) {return x.name_ < y.name_;} + + friend DATE_API std::ostream& operator<<(std::ostream& os, const link& x); +}; + +inline bool operator!=(const link& x, const link& y) {return !(x == y);} +inline bool operator> (const link& x, const link& y) {return y < x;} +inline bool operator<=(const link& x, const link& y) {return !(y < x);} +inline bool operator>=(const link& x, const link& y) {return !(x < y);} + +#endif // !USE_OS_TZDB + +#if !MISSING_LEAP_SECONDS + +class leap +{ +private: + sys_seconds date_; + +public: +#if USE_OS_TZDB + DATE_API explicit leap(const sys_seconds& s, detail::undocumented); +#else + DATE_API explicit leap(const std::string& s, detail::undocumented); +#endif + + sys_seconds date() const {return date_;} + + friend bool operator==(const leap& x, const leap& y) {return x.date_ == y.date_;} + friend bool operator< (const leap& x, const leap& y) {return x.date_ < y.date_;} + + template + friend + bool + operator==(const leap& x, const sys_time& y) + { + return x.date_ == y; + } + + template + friend + bool + operator< (const leap& x, const sys_time& y) + { + return x.date_ < y; + } + + template + friend + bool + operator< (const sys_time& x, const leap& y) + { + return x < y.date_; + } + + friend DATE_API std::ostream& operator<<(std::ostream& os, const leap& x); +}; + +inline bool operator!=(const leap& x, const leap& y) {return !(x == y);} +inline bool operator> (const leap& x, const leap& y) {return y < x;} +inline bool operator<=(const leap& x, const leap& y) {return !(y < x);} +inline bool operator>=(const leap& x, const leap& y) {return !(x < y);} + +template +inline +bool +operator==(const sys_time& x, const leap& y) +{ + return y == x; +} + +template +inline +bool +operator!=(const leap& x, const sys_time& y) +{ + return !(x == y); +} + +template +inline +bool +operator!=(const sys_time& x, const leap& y) +{ + return !(x == y); +} + +template +inline +bool +operator> (const leap& x, const sys_time& y) +{ + return y < x; +} + +template +inline +bool +operator> (const sys_time& x, const leap& y) +{ + return y < x; +} + +template +inline +bool +operator<=(const leap& x, const sys_time& y) +{ + return !(y < x); +} + +template +inline +bool +operator<=(const sys_time& x, const leap& y) +{ + return !(y < x); +} + +template +inline +bool +operator>=(const leap& x, const sys_time& y) +{ + return !(x < y); +} + +template +inline +bool +operator>=(const sys_time& x, const leap& y) +{ + return !(x < y); +} + +#endif // !MISSING_LEAP_SECONDS + +#ifdef _WIN32 + +namespace detail +{ + +// The time zone mapping is modelled after this data file: +// http://unicode.org/repos/cldr/trunk/common/supplemental/windowsZones.xml +// and the field names match the element names from the mapZone element +// of windowsZones.xml. +// The website displays this file here: +// http://www.unicode.org/cldr/charts/latest/supplemental/zone_tzid.html +// The html view is sorted before being displayed but is otherwise the same +// There is a mapping between the os centric view (in this case windows) +// the html displays uses and the generic view the xml file. +// That mapping is this: +// display column "windows" -> xml field "other". +// display column "region" -> xml field "territory". +// display column "tzid" -> xml field "type". +// This structure uses the generic terminology because it could be +// used to to support other os/native name conversions, not just windows, +// and using the same generic names helps retain the connection to the +// origin of the data that we are using. +struct timezone_mapping +{ + timezone_mapping(const char* other, const char* territory, const char* type) + : other(other), territory(territory), type(type) + { + } + timezone_mapping() = default; + std::string other; + std::string territory; + std::string type; +}; + +} // detail + +#endif // _WIN32 + +struct tzdb +{ + std::string version = "unknown"; + std::vector zones; +#if !USE_OS_TZDB + std::vector links; +#endif +#if !MISSING_LEAP_SECONDS + std::vector leaps; +#endif +#if !USE_OS_TZDB + std::vector rules; +#endif +#ifdef _WIN32 + std::vector mappings; +#endif + tzdb* next = nullptr; + + tzdb() = default; +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) + tzdb(tzdb&&) = default; + tzdb& operator=(tzdb&&) = default; +#else // defined(_MSC_VER) && (_MSC_VER < 1900) + tzdb(tzdb&& src) + : version(std::move(src.version)) + , zones(std::move(src.zones)) + , links(std::move(src.links)) + , leaps(std::move(src.leaps)) + , rules(std::move(src.rules)) + , mappings(std::move(src.mappings)) + {} + + tzdb& operator=(tzdb&& src) + { + version = std::move(src.version); + zones = std::move(src.zones); + links = std::move(src.links); + leaps = std::move(src.leaps); + rules = std::move(src.rules); + mappings = std::move(src.mappings); + return *this; + } +#endif // defined(_MSC_VER) && (_MSC_VER < 1900) + +#if HAS_STRING_VIEW + const time_zone* locate_zone(std::string_view tz_name) const; +#else + const time_zone* locate_zone(const std::string& tz_name) const; +#endif + const time_zone* current_zone() const; +}; + +using TZ_DB = tzdb; + +DATE_API std::ostream& +operator<<(std::ostream& os, const tzdb& db); + +DATE_API const tzdb& get_tzdb(); + +class tzdb_list +{ + std::atomic head_{nullptr}; + +public: + ~tzdb_list(); + tzdb_list() = default; + tzdb_list(tzdb_list&& x) noexcept; + + const tzdb& front() const noexcept {return *head_;} + tzdb& front() noexcept {return *head_;} + + class const_iterator; + + const_iterator begin() const noexcept; + const_iterator end() const noexcept; + + const_iterator cbegin() const noexcept; + const_iterator cend() const noexcept; + + const_iterator erase_after(const_iterator p) noexcept; + + struct undocumented_helper; +private: + void push_front(tzdb* tzdb) noexcept; +}; + +class tzdb_list::const_iterator +{ + tzdb* p_ = nullptr; + + explicit const_iterator(tzdb* p) noexcept : p_{p} {} +public: + const_iterator() = default; + + using iterator_category = std::forward_iterator_tag; + using value_type = tzdb; + using reference = const value_type&; + using pointer = const value_type*; + using difference_type = std::ptrdiff_t; + + reference operator*() const noexcept {return *p_;} + pointer operator->() const noexcept {return p_;} + + const_iterator& operator++() noexcept {p_ = p_->next; return *this;} + const_iterator operator++(int) noexcept {auto t = *this; ++(*this); return t;} + + friend + bool + operator==(const const_iterator& x, const const_iterator& y) noexcept + {return x.p_ == y.p_;} + + friend + bool + operator!=(const const_iterator& x, const const_iterator& y) noexcept + {return !(x == y);} + + friend class tzdb_list; +}; + +inline +tzdb_list::const_iterator +tzdb_list::begin() const noexcept +{ + return const_iterator{head_}; +} + +inline +tzdb_list::const_iterator +tzdb_list::end() const noexcept +{ + return const_iterator{nullptr}; +} + +inline +tzdb_list::const_iterator +tzdb_list::cbegin() const noexcept +{ + return begin(); +} + +inline +tzdb_list::const_iterator +tzdb_list::cend() const noexcept +{ + return end(); +} + +DATE_API tzdb_list& get_tzdb_list(); + +#if !USE_OS_TZDB + +DATE_API const tzdb& reload_tzdb(); +DATE_API void set_install(const std::string& install); + +#endif // !USE_OS_TZDB + +#if HAS_REMOTE_API + +DATE_API std::string remote_version(); +DATE_API bool remote_download(const std::string& version); +DATE_API bool remote_install(const std::string& version); + +#endif + +// zoned_time + +namespace detail +{ + +template +inline +T* +to_raw_pointer(T* p) noexcept +{ + return p; +} + +template +inline +auto +to_raw_pointer(Pointer p) noexcept + -> decltype(detail::to_raw_pointer(p.operator->())) +{ + return detail::to_raw_pointer(p.operator->()); +} + +} // namespace detail + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time() + : zone_(zoned_traits::default_zone()) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const sys_time& st) + : zone_(zoned_traits::default_zone()) + , tp_(st) + {} + +template +inline +zoned_time::zoned_time(TimeZonePtr z) + : zone_(std::move(z)) + {assert(detail::to_raw_pointer(zone_) != nullptr);} + +#if HAS_STRING_VIEW + +template +template +inline +zoned_time::zoned_time(std::string_view name) + : zoned_time(zoned_traits::locate_zone(name)) + {} + +#else // !HAS_STRING_VIEW + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const std::string& name) + : zoned_time(zoned_traits::locate_zone(name)) + {} + +#endif // !HAS_STRING_VIEW + +template +template +inline +zoned_time::zoned_time(const zoned_time& zt) NOEXCEPT + : zone_(zt.zone_) + , tp_(zt.tp_) + {} + +template +inline +zoned_time::zoned_time(TimeZonePtr z, const sys_time& st) + : zone_(std::move(z)) + , tp_(st) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(TimeZonePtr z, const local_time& t) + : zone_(std::move(z)) + , tp_(zone_->to_sys(t)) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(TimeZonePtr z, const local_time& t, + choose c) + : zone_(std::move(z)) + , tp_(zone_->to_sys(t, c)) + {} + +template +template +inline +zoned_time::zoned_time(TimeZonePtr z, + const zoned_time& zt) + : zone_(std::move(z)) + , tp_(zt.tp_) + {} + +template +template +inline +zoned_time::zoned_time(TimeZonePtr z, + const zoned_time& zt, choose) + : zoned_time(std::move(z), zt) + {} + +#if HAS_STRING_VIEW + +template +template +inline +zoned_time::zoned_time(std::string_view name, + const sys_time& st) + : zoned_time(zoned_traits::locate_zone(name), st) + {} + +template +template +inline +zoned_time::zoned_time(std::string_view name, + const local_time& t) + : zoned_time(zoned_traits::locate_zone(name), t) + {} + +template +template +inline +zoned_time::zoned_time(std::string_view name, + const local_time& t, choose c) + : zoned_time(zoned_traits::locate_zone(name), t, c) + {} + +template +template +inline +zoned_time::zoned_time(std::string_view name, const zoned_time& zt) + : zoned_time(zoned_traits::locate_zone(name), zt) + {} + +template +template +inline +zoned_time::zoned_time(std::string_view name, + const zoned_time& zt, choose c) + : zoned_time(zoned_traits::locate_zone(name), zt, c) + {} + +#else // !HAS_STRING_VIEW + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const std::string& name, + const sys_time& st) + : zoned_time(zoned_traits::locate_zone(name), st) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const char* name, + const sys_time& st) + : zoned_time(zoned_traits::locate_zone(name), st) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const std::string& name, + const local_time& t) + : zoned_time(zoned_traits::locate_zone(name), t) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const char* name, + const local_time& t) + : zoned_time(zoned_traits::locate_zone(name), t) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const std::string& name, + const local_time& t, choose c) + : zoned_time(zoned_traits::locate_zone(name), t, c) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const char* name, + const local_time& t, choose c) + : zoned_time(zoned_traits::locate_zone(name), t, c) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const std::string& name, + const zoned_time& zt) + : zoned_time(zoned_traits::locate_zone(name), zt) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const char* name, const zoned_time& zt) + : zoned_time(zoned_traits::locate_zone(name), zt) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const std::string& name, + const zoned_time& zt, choose c) + : zoned_time(zoned_traits::locate_zone(name), zt, c) + {} + +template +#if !defined(_MSC_VER) || (_MSC_VER > 1900) +template +#endif +inline +zoned_time::zoned_time(const char* name, + const zoned_time& zt, choose c) + : zoned_time(zoned_traits::locate_zone(name), zt, c) + {} + +#endif // HAS_STRING_VIEW + +template +inline +zoned_time& +zoned_time::operator=(const sys_time& st) +{ + tp_ = st; + return *this; +} + +template +inline +zoned_time& +zoned_time::operator=(const local_time& ut) +{ + tp_ = zone_->to_sys(ut); + return *this; +} + +template +inline +zoned_time::operator local_time::duration>() const +{ + return get_local_time(); +} + +template +inline +zoned_time::operator sys_time::duration>() const +{ + return get_sys_time(); +} + +template +inline +TimeZonePtr +zoned_time::get_time_zone() const +{ + return zone_; +} + +template +inline +local_time::duration> +zoned_time::get_local_time() const +{ + return zone_->to_local(tp_); +} + +template +inline +sys_time::duration> +zoned_time::get_sys_time() const +{ + return tp_; +} + +template +inline +sys_info +zoned_time::get_info() const +{ + return zone_->get_info(tp_); +} + +// make_zoned_time + +inline +zoned_time +make_zoned() +{ + return zoned_time(); +} + +template +inline +zoned_time::type> +make_zoned(const sys_time& tp) +{ + return zoned_time::type>(tp); +} + +template 1900) + , class = typename std::enable_if + < + std::is_class + < + typename std::decay + < + decltype(*detail::to_raw_pointer(std::declval())) + >::type + >{} + >::type +#endif + > +inline +zoned_time +make_zoned(TimeZonePtr z) +{ + return zoned_time(std::move(z)); +} + +inline +zoned_seconds +make_zoned(const std::string& name) +{ + return zoned_seconds(name); +} + +template 1900) + , class = typename std::enable_if + < + std::is_class())>::type>{} + >::type +#endif + > +inline +zoned_time::type, TimeZonePtr> +make_zoned(TimeZonePtr zone, const local_time& tp) +{ + return zoned_time::type, + TimeZonePtr>(std::move(zone), tp); +} + +template 1900) + , class = typename std::enable_if + < + std::is_class())>::type>{} + >::type +#endif + > +inline +zoned_time::type, TimeZonePtr> +make_zoned(TimeZonePtr zone, const local_time& tp, choose c) +{ + return zoned_time::type, + TimeZonePtr>(std::move(zone), tp, c); +} + +template +inline +zoned_time::type> +make_zoned(const std::string& name, const local_time& tp) +{ + return zoned_time::type>(name, tp); +} + +template +inline +zoned_time::type> +make_zoned(const std::string& name, const local_time& tp, choose c) +{ + return zoned_time::type>(name, tp, c); +} + +template +inline +zoned_time +make_zoned(TimeZonePtr zone, const zoned_time& zt) +{ + return zoned_time(std::move(zone), zt); +} + +template +inline +zoned_time +make_zoned(const std::string& name, const zoned_time& zt) +{ + return zoned_time(name, zt); +} + +template +inline +zoned_time +make_zoned(TimeZonePtr zone, const zoned_time& zt, choose c) +{ + return zoned_time(std::move(zone), zt, c); +} + +template +inline +zoned_time +make_zoned(const std::string& name, const zoned_time& zt, choose c) +{ + return zoned_time(name, zt, c); +} + +template 1900) + , class = typename std::enable_if + < + std::is_class())>::type>{} + >::type +#endif + > +inline +zoned_time::type, TimeZonePtr> +make_zoned(TimeZonePtr zone, const sys_time& st) +{ + return zoned_time::type, + TimeZonePtr>(std::move(zone), st); +} + +template +inline +zoned_time::type> +make_zoned(const std::string& name, const sys_time& st) +{ + return zoned_time::type>(name, st); +} + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const zoned_time& tp) +{ + using duration = typename zoned_time::duration; + using LT = local_time; + auto const tz = tp.get_time_zone(); + auto const st = tp.get_sys_time(); + auto const info = tz->get_info(st); + return to_stream(os, fmt, LT{(st+info.offset).time_since_epoch()}, + &info.abbrev, &info.offset); +} + +template +inline +std::basic_ostream& +operator<<(std::basic_ostream& os, const zoned_time& t) +{ + const CharT fmt[] = {'%', 'F', ' ', '%', 'T', ' ', '%', 'Z', CharT{}}; + return to_stream(os, fmt, t); +} + +#if !MISSING_LEAP_SECONDS + +class utc_clock +{ +public: + using duration = std::chrono::system_clock::duration; + using rep = duration::rep; + using period = duration::period; + using time_point = std::chrono::time_point; + static CONSTDATA bool is_steady = false; + + static time_point now(); + + template + static + std::chrono::time_point::type> + to_sys(const std::chrono::time_point&); + + template + static + std::chrono::time_point::type> + from_sys(const std::chrono::time_point&); +}; + +template + using utc_time = std::chrono::time_point; + +using utc_seconds = utc_time; + +template +utc_time::type> +utc_clock::from_sys(const sys_time& st) +{ + using namespace std::chrono; + using duration = typename std::common_type::type; + auto const& leaps = get_tzdb().leaps; + auto const lt = std::upper_bound(leaps.begin(), leaps.end(), st); + return utc_time{st.time_since_epoch() + seconds{lt-leaps.begin()}}; +} + +// Return pair +// first is true if ut is during a leap second insertion, otherwise false. +// If ut is during a leap second insertion, that leap second is included in the count +template +std::pair +is_leap_second(date::utc_time const& ut) +{ + using namespace date; + using namespace std::chrono; + using duration = typename std::common_type::type; + auto const& leaps = get_tzdb().leaps; + auto tp = sys_time{ut.time_since_epoch()}; + auto const lt = std::upper_bound(leaps.begin(), leaps.end(), tp); + auto ds = seconds{lt-leaps.begin()}; + tp -= ds; + auto ls = false; + if (lt > leaps.begin()) + { + if (tp < lt[-1]) + { + if (tp >= lt[-1].date() - seconds{1}) + ls = true; + else + --ds; + } + } + return {ls, ds}; +} + +template +sys_time::type> +utc_clock::to_sys(const utc_time& ut) +{ + using namespace std::chrono; + using duration = typename std::common_type::type; + auto ls = is_leap_second(ut); + auto tp = sys_time{ut.time_since_epoch() - ls.second}; + if (ls.first) + tp = floor(tp) + seconds{1} - duration{1}; + return tp; +} + +inline +utc_clock::time_point +utc_clock::now() +{ + using namespace std::chrono; + return from_sys(system_clock::now()); +} + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const utc_time& t) +{ + using namespace std; + using namespace std::chrono; + using CT = typename common_type::type; + const string abbrev("UTC"); + CONSTDATA seconds offset{0}; + auto ls = is_leap_second(t); + auto tp = sys_time{t.time_since_epoch() - ls.second}; + auto const sd = floor(tp); + year_month_day ymd = sd; + auto time = make_time(tp - sys_seconds{sd}); + time.seconds() += seconds{ls.first}; + fields fds{ymd, time}; + return to_stream(os, fmt, fds, &abbrev, &offset); +} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const utc_time& t) +{ + const CharT fmt[] = {'%', 'F', ' ', '%', 'T', CharT{}}; + return to_stream(os, fmt, t); +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + utc_time& tp, std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = typename common_type::type; + minutes offset_local{}; + auto offptr = offset ? offset : &offset_local; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offptr); + if (!fds.ymd.ok()) + is.setstate(ios::failbit); + if (!is.fail()) + { + bool is_60_sec = fds.tod.seconds() == seconds{60}; + if (is_60_sec) + fds.tod.seconds() -= seconds{1}; + auto tmp = utc_clock::from_sys(sys_days(fds.ymd) - *offptr + fds.tod.to_duration()); + if (is_60_sec) + tmp += seconds{1}; + if (is_60_sec != is_leap_second(tmp).first || !fds.tod.in_conventional_range()) + { + is.setstate(ios::failbit); + return is; + } + tp = time_point_cast(tmp); + } + return is; +} + +// tai_clock + +class tai_clock +{ +public: + using duration = std::chrono::system_clock::duration; + using rep = duration::rep; + using period = duration::period; + using time_point = std::chrono::time_point; + static const bool is_steady = false; + + static time_point now(); + + template + static + std::chrono::time_point::type> + to_utc(const std::chrono::time_point&) NOEXCEPT; + + template + static + std::chrono::time_point::type> + from_utc(const std::chrono::time_point&) NOEXCEPT; +}; + +template + using tai_time = std::chrono::time_point; + +using tai_seconds = tai_time; + +template +inline +utc_time::type> +tai_clock::to_utc(const tai_time& t) NOEXCEPT +{ + using namespace std::chrono; + using duration = typename std::common_type::type; + return utc_time{t.time_since_epoch()} - + (sys_days(year{1970}/jan/1) - sys_days(year{1958}/jan/1) + seconds{10}); +} + +template +inline +tai_time::type> +tai_clock::from_utc(const utc_time& t) NOEXCEPT +{ + using namespace std::chrono; + using duration = typename std::common_type::type; + return tai_time{t.time_since_epoch()} + + (sys_days(year{1970}/jan/1) - sys_days(year{1958}/jan/1) + seconds{10}); +} + +inline +tai_clock::time_point +tai_clock::now() +{ + using namespace std::chrono; + return from_utc(utc_clock::now()); +} + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const tai_time& t) +{ + using namespace std; + using namespace std::chrono; + using CT = typename common_type::type; + const string abbrev("TAI"); + CONSTDATA seconds offset{0}; + auto tp = sys_time{t.time_since_epoch()} - + seconds(sys_days(year{1970}/jan/1) - sys_days(year{1958}/jan/1)); + auto const sd = floor(tp); + year_month_day ymd = sd; + auto time = make_time(tp - sys_seconds{sd}); + fields fds{ymd, time}; + return to_stream(os, fmt, fds, &abbrev, &offset); +} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const tai_time& t) +{ + const CharT fmt[] = {'%', 'F', ' ', '%', 'T', CharT{}}; + return to_stream(os, fmt, t); +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + tai_time& tp, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = typename common_type::type; + minutes offset_local{}; + auto offptr = offset ? offset : &offset_local; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offptr); + if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) + is.setstate(ios::failbit); + if (!is.fail()) + tp = tai_time{duration_cast( + (sys_days(fds.ymd) + + (sys_days(year{1970}/jan/1) - sys_days(year{1958}/jan/1)) - + *offptr + fds.tod.to_duration()).time_since_epoch())}; + return is; +} + +// gps_clock + +class gps_clock +{ +public: + using duration = std::chrono::system_clock::duration; + using rep = duration::rep; + using period = duration::period; + using time_point = std::chrono::time_point; + static const bool is_steady = false; + + static time_point now(); + + template + static + std::chrono::time_point::type> + to_utc(const std::chrono::time_point&) NOEXCEPT; + + template + static + std::chrono::time_point::type> + from_utc(const std::chrono::time_point&) NOEXCEPT; + +}; + +template + using gps_time = std::chrono::time_point; + +using gps_seconds = gps_time; + +template +inline +utc_time::type> +gps_clock::to_utc(const gps_time& t) NOEXCEPT +{ + using namespace std::chrono; + using duration = typename std::common_type::type; + return utc_time{t.time_since_epoch()} + + (sys_days(year{1980}/jan/sun[1]) - sys_days(year{1970}/jan/1) + seconds{9}); +} + +template +inline +gps_time::type> +gps_clock::from_utc(const utc_time& t) NOEXCEPT +{ + using namespace std::chrono; + using duration = typename std::common_type::type; + return gps_time{t.time_since_epoch()} - + (sys_days(year{1980}/jan/sun[1]) - sys_days(year{1970}/jan/1) + seconds{9}); +} + +inline +gps_clock::time_point +gps_clock::now() +{ + using namespace std::chrono; + return from_utc(utc_clock::now()); +} + +template +std::basic_ostream& +to_stream(std::basic_ostream& os, const CharT* fmt, + const gps_time& t) +{ + using namespace std; + using namespace std::chrono; + using CT = typename common_type::type; + const string abbrev("GPS"); + CONSTDATA seconds offset{0}; + auto tp = sys_time{t.time_since_epoch()} + + seconds(sys_days(year{1980}/jan/sun[1]) - sys_days(year{1970}/jan/1)); + auto const sd = floor(tp); + year_month_day ymd = sd; + auto time = make_time(tp - sys_seconds{sd}); + fields fds{ymd, time}; + return to_stream(os, fmt, fds, &abbrev, &offset); +} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const gps_time& t) +{ + const CharT fmt[] = {'%', 'F', ' ', '%', 'T', CharT{}}; + return to_stream(os, fmt, t); +} + +template > +std::basic_istream& +from_stream(std::basic_istream& is, const CharT* fmt, + gps_time& tp, + std::basic_string* abbrev = nullptr, + std::chrono::minutes* offset = nullptr) +{ + using namespace std; + using namespace std::chrono; + using CT = typename common_type::type; + minutes offset_local{}; + auto offptr = offset ? offset : &offset_local; + fields fds{}; + from_stream(is, fmt, fds, abbrev, offptr); + if (!fds.ymd.ok() || !fds.tod.in_conventional_range()) + is.setstate(ios::failbit); + if (!is.fail()) + tp = gps_time{duration_cast( + (sys_days(fds.ymd) - + (sys_days(year{1980}/jan/sun[1]) - sys_days(year{1970}/jan/1)) - + *offptr + fds.tod.to_duration()).time_since_epoch())}; + return is; +} + +// clock_time_conversion + +template +struct clock_time_conversion +{}; + +template <> +struct clock_time_conversion +{ + template + sys_time + operator()(const sys_time& st) const + { + return st; + } +}; + +template <> +struct clock_time_conversion +{ + template + utc_time + operator()(const utc_time& ut) const + { + return ut; + } +}; + +template <> +struct clock_time_conversion +{ + template + utc_time::type> + operator()(const sys_time& st) const + { + return utc_clock::from_sys(st); + } +}; + +template <> +struct clock_time_conversion +{ + template + sys_time::type> + operator()(const utc_time& ut) const + { + return utc_clock::to_sys(ut); + } +}; + +template +struct clock_time_conversion +{ + template + std::chrono::time_point + operator()(const std::chrono::time_point& tp) const + { + return tp; + } +}; + +namespace ctc_detail +{ + +template + using time_point = std::chrono::time_point; + +using std::declval; +using std::chrono::system_clock; + +//Check if TimePoint is time for given clock, +//if not emits hard error +template +struct return_clock_time +{ + using clock_time_point = time_point; + using type = TimePoint; + + static_assert(std::is_same::value, + "time point with appropariate clock shall be returned"); +}; + +// Check if Clock has to_sys method accepting TimePoint with given duration const& and +// returning sys_time. If so has nested type member equal to return type to_sys. +template +struct return_to_sys +{}; + +template +struct return_to_sys + < + Clock, Duration, + decltype(Clock::to_sys(declval const&>()), void()) + > + : return_clock_time + < + system_clock, + decltype(Clock::to_sys(declval const&>())) + > +{}; + +// Similiar to above +template +struct return_from_sys +{}; + +template +struct return_from_sys + < + Clock, Duration, + decltype(Clock::from_sys(declval const&>()), + void()) + > + : return_clock_time + < + Clock, + decltype(Clock::from_sys(declval const&>())) + > +{}; + +// Similiar to above +template +struct return_to_utc +{}; + +template +struct return_to_utc + < + Clock, Duration, + decltype(Clock::to_utc(declval const&>()), void()) + > + : return_clock_time + < + utc_clock, + decltype(Clock::to_utc(declval const&>()))> +{}; + +// Similiar to above +template +struct return_from_utc +{}; + +template +struct return_from_utc + < + Clock, Duration, + decltype(Clock::from_utc(declval const&>()), + void()) + > + : return_clock_time + < + Clock, + decltype(Clock::from_utc(declval const&>())) + > +{}; + +} // namespace ctc_detail + +template +struct clock_time_conversion +{ + template + typename ctc_detail::return_to_sys::type + operator()(const std::chrono::time_point& tp) const + { + return SrcClock::to_sys(tp); + } +}; + +template +struct clock_time_conversion +{ + template + typename ctc_detail::return_from_sys::type + operator()(const sys_time& st) const + { + return DstClock::from_sys(st); + } +}; + +template +struct clock_time_conversion +{ + template + typename ctc_detail::return_to_utc::type + operator()(const std::chrono::time_point& tp) const + { + return SrcClock::to_utc(tp); + } +}; + +template +struct clock_time_conversion +{ + template + typename ctc_detail::return_from_utc::type + operator()(const utc_time& ut) const + { + return DstClock::from_utc(ut); + } +}; + +namespace clock_cast_detail +{ + +template + using time_point = std::chrono::time_point; +using std::chrono::system_clock; + +template +auto +conv_clock(const time_point& t) + -> decltype(std::declval>()(t)) +{ + return clock_time_conversion{}(t); +} + +//direct trait conversion, 1st candidate +template +auto +cc_impl(const time_point& t, const time_point*) + -> decltype(conv_clock(t)) +{ + return conv_clock(t); +} + +//conversion through sys, 2nd candidate +template +auto +cc_impl(const time_point& t, const void*) + -> decltype(conv_clock(conv_clock(t))) +{ + return conv_clock(conv_clock(t)); +} + +//conversion through utc, 2nd candidate +template +auto +cc_impl(const time_point& t, const void*) + -> decltype(0, // MSVC_WORKAROUND + conv_clock(conv_clock(t))) +{ + return conv_clock(conv_clock(t)); +} + +//conversion through sys and utc, 3rd candidate +template +auto +cc_impl(const time_point& t, ...) + -> decltype(conv_clock(conv_clock(conv_clock(t)))) +{ + return conv_clock(conv_clock(conv_clock(t))); +} + +//conversion through utc and sys, 3rd candidate +template +auto +cc_impl(const time_point& t, ...) + -> decltype(0, // MSVC_WORKAROUND + conv_clock(conv_clock(conv_clock(t)))) +{ + return conv_clock(conv_clock(conv_clock(t))); +} + +} // namespace clock_cast_detail + +template +auto +clock_cast(const std::chrono::time_point& tp) + -> decltype(clock_cast_detail::cc_impl(tp, &tp)) +{ + return clock_cast_detail::cc_impl(tp, &tp); +} + +// Deprecated API + +template +inline +sys_time::type> +to_sys_time(const utc_time& t) +{ + return utc_clock::to_sys(t); +} + +template +inline +sys_time::type> +to_sys_time(const tai_time& t) +{ + return utc_clock::to_sys(tai_clock::to_utc(t)); +} + +template +inline +sys_time::type> +to_sys_time(const gps_time& t) +{ + return utc_clock::to_sys(gps_clock::to_utc(t)); +} + + +template +inline +utc_time::type> +to_utc_time(const sys_time& t) +{ + return utc_clock::from_sys(t); +} + +template +inline +utc_time::type> +to_utc_time(const tai_time& t) +{ + return tai_clock::to_utc(t); +} + +template +inline +utc_time::type> +to_utc_time(const gps_time& t) +{ + return gps_clock::to_utc(t); +} + + +template +inline +tai_time::type> +to_tai_time(const sys_time& t) +{ + return tai_clock::from_utc(utc_clock::from_sys(t)); +} + +template +inline +tai_time::type> +to_tai_time(const utc_time& t) +{ + return tai_clock::from_utc(t); +} + +template +inline +tai_time::type> +to_tai_time(const gps_time& t) +{ + return tai_clock::from_utc(gps_clock::to_utc(t)); +} + + +template +inline +gps_time::type> +to_gps_time(const sys_time& t) +{ + return gps_clock::from_utc(utc_clock::from_sys(t)); +} + +template +inline +gps_time::type> +to_gps_time(const utc_time& t) +{ + return gps_clock::from_utc(t); +} + +template +inline +gps_time::type> +to_gps_time(const tai_time& t) +{ + return gps_clock::from_utc(tai_clock::to_utc(t)); +} + +#endif // !MISSING_LEAP_SECONDS + +} // namespace date +} // namespace util +} // namespace arrow + +#endif // TZ_H diff --git a/cpp/src/arrow/vendored/datetime/tz_private.h b/cpp/src/arrow/vendored/datetime/tz_private.h new file mode 100644 index 0000000000000..cafa5ea2bf1a8 --- /dev/null +++ b/cpp/src/arrow/vendored/datetime/tz_private.h @@ -0,0 +1,324 @@ +#ifndef TZ_PRIVATE_H +#define TZ_PRIVATE_H + +// The MIT License (MIT) +// +// Copyright (c) 2015, 2016 Howard Hinnant +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Our apologies. When the previous paragraph was written, lowercase had not yet +// been invented (that would involve another several millennia of evolution). +// We did not mean to shout. + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +#include "tz.h" +#else +#include "date.h" +#include +#endif + +namespace arrow +{ +namespace util +{ +namespace date +{ + +namespace detail +{ + +#if !USE_OS_TZDB + +enum class tz {utc, local, standard}; + +//forward declare to avoid warnings in gcc 6.2 +class MonthDayTime; +std::istream& operator>>(std::istream& is, MonthDayTime& x); +std::ostream& operator<<(std::ostream& os, const MonthDayTime& x); + + +class MonthDayTime +{ +private: + struct pair + { +#if defined(_MSC_VER) && (_MSC_VER < 1900) + pair() : month_day_(date::jan / 1), weekday_(0U) {} + + pair(const date::month_day& month_day, const date::weekday& weekday) + : month_day_(month_day), weekday_(weekday) {} +#endif + + date::month_day month_day_; + date::weekday weekday_; + }; + + enum Type {month_day, month_last_dow, lteq, gteq}; + + Type type_{month_day}; + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) + union U +#else + struct U +#endif + { + date::month_day month_day_; + date::month_weekday_last month_weekday_last_; + pair month_day_weekday_; + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) + U() : month_day_{date::jan/1} {} +#else + U() : + month_day_(date::jan/1), + month_weekday_last_(date::month(0U), date::weekday_last(date::weekday(0U))) + {} + +#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) + + U& operator=(const date::month_day& x); + U& operator=(const date::month_weekday_last& x); + U& operator=(const pair& x); + } u; + + std::chrono::hours h_{0}; + std::chrono::minutes m_{0}; + std::chrono::seconds s_{0}; + tz zone_{tz::local}; + +public: + MonthDayTime() = default; + MonthDayTime(local_seconds tp, tz timezone); + MonthDayTime(const date::month_day& md, tz timezone); + + date::day day() const; + date::month month() const; + tz zone() const {return zone_;} + + void canonicalize(date::year y); + + sys_seconds + to_sys(date::year y, std::chrono::seconds offset, std::chrono::seconds save) const; + sys_days to_sys_days(date::year y) const; + + sys_seconds to_time_point(date::year y) const; + int compare(date::year y, const MonthDayTime& x, date::year yx, + std::chrono::seconds offset, std::chrono::minutes prev_save) const; + + friend std::istream& operator>>(std::istream& is, MonthDayTime& x); + friend std::ostream& operator<<(std::ostream& os, const MonthDayTime& x); +}; + +// A Rule specifies one or more set of datetimes without using an offset. +// Multiple dates are specified with multiple years. The years in effect +// go from starting_year_ to ending_year_, inclusive. starting_year_ <= +// ending_year_. save_ is in effect for times from the specified time +// onward, including the specified time. When the specified time is +// local, it uses the save_ from the chronologically previous Rule, or if +// there is none, 0. + +//forward declare to avoid warnings in gcc 6.2 +class Rule; +bool operator==(const Rule& x, const Rule& y); +bool operator<(const Rule& x, const Rule& y); +bool operator==(const Rule& x, const date::year& y); +bool operator<(const Rule& x, const date::year& y); +bool operator==(const date::year& x, const Rule& y); +bool operator<(const date::year& x, const Rule& y); +bool operator==(const Rule& x, const std::string& y); +bool operator<(const Rule& x, const std::string& y); +bool operator==(const std::string& x, const Rule& y); +bool operator<(const std::string& x, const Rule& y); +std::ostream& operator<<(std::ostream& os, const Rule& r); + +class Rule +{ +private: + std::string name_; + date::year starting_year_{0}; + date::year ending_year_{0}; + MonthDayTime starting_at_; + std::chrono::minutes save_{0}; + std::string abbrev_; + +public: + Rule() = default; + explicit Rule(const std::string& s); + Rule(const Rule& r, date::year starting_year, date::year ending_year); + + const std::string& name() const {return name_;} + const std::string& abbrev() const {return abbrev_;} + + const MonthDayTime& mdt() const {return starting_at_;} + const date::year& starting_year() const {return starting_year_;} + const date::year& ending_year() const {return ending_year_;} + const std::chrono::minutes& save() const {return save_;} + + static void split_overlaps(std::vector& rules); + + friend bool operator==(const Rule& x, const Rule& y); + friend bool operator<(const Rule& x, const Rule& y); + friend bool operator==(const Rule& x, const date::year& y); + friend bool operator<(const Rule& x, const date::year& y); + friend bool operator==(const date::year& x, const Rule& y); + friend bool operator<(const date::year& x, const Rule& y); + friend bool operator==(const Rule& x, const std::string& y); + friend bool operator<(const Rule& x, const std::string& y); + friend bool operator==(const std::string& x, const Rule& y); + friend bool operator<(const std::string& x, const Rule& y); + + friend std::ostream& operator<<(std::ostream& os, const Rule& r); + +private: + date::day day() const; + date::month month() const; + static void split_overlaps(std::vector& rules, std::size_t i, std::size_t& e); + static bool overlaps(const Rule& x, const Rule& y); + static void split(std::vector& rules, std::size_t i, std::size_t k, + std::size_t& e); +}; + +inline bool operator!=(const Rule& x, const Rule& y) {return !(x == y);} +inline bool operator> (const Rule& x, const Rule& y) {return y < x;} +inline bool operator<=(const Rule& x, const Rule& y) {return !(y < x);} +inline bool operator>=(const Rule& x, const Rule& y) {return !(x < y);} + +inline bool operator!=(const Rule& x, const date::year& y) {return !(x == y);} +inline bool operator> (const Rule& x, const date::year& y) {return y < x;} +inline bool operator<=(const Rule& x, const date::year& y) {return !(y < x);} +inline bool operator>=(const Rule& x, const date::year& y) {return !(x < y);} + +inline bool operator!=(const date::year& x, const Rule& y) {return !(x == y);} +inline bool operator> (const date::year& x, const Rule& y) {return y < x;} +inline bool operator<=(const date::year& x, const Rule& y) {return !(y < x);} +inline bool operator>=(const date::year& x, const Rule& y) {return !(x < y);} + +inline bool operator!=(const Rule& x, const std::string& y) {return !(x == y);} +inline bool operator> (const Rule& x, const std::string& y) {return y < x;} +inline bool operator<=(const Rule& x, const std::string& y) {return !(y < x);} +inline bool operator>=(const Rule& x, const std::string& y) {return !(x < y);} + +inline bool operator!=(const std::string& x, const Rule& y) {return !(x == y);} +inline bool operator> (const std::string& x, const Rule& y) {return y < x;} +inline bool operator<=(const std::string& x, const Rule& y) {return !(y < x);} +inline bool operator>=(const std::string& x, const Rule& y) {return !(x < y);} + +struct zonelet +{ + enum tag {has_rule, has_save, is_empty}; + + std::chrono::seconds gmtoff_; + tag tag_ = has_rule; + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) + union U +#else + struct U +#endif + { + std::string rule_; + std::chrono::minutes save_; + + ~U() {} + U() {} + U(const U&) {} + U& operator=(const U&) = delete; + } u; + + std::string format_; + date::year until_year_{0}; + MonthDayTime until_date_; + sys_seconds until_utc_; + local_seconds until_std_; + local_seconds until_loc_; + std::chrono::minutes initial_save_{}; + std::string initial_abbrev_; + std::pair first_rule_{nullptr, date::year::min()}; + std::pair last_rule_{nullptr, date::year::max()}; + + ~zonelet(); + zonelet(); + zonelet(const zonelet& i); + zonelet& operator=(const zonelet&) = delete; +}; + +#else // USE_OS_TZDB + +struct ttinfo +{ + std::int32_t tt_gmtoff; + unsigned char tt_isdst; + unsigned char tt_abbrind; + unsigned char pad[2]; +}; + +static_assert(sizeof(ttinfo) == 8, ""); + +struct expanded_ttinfo +{ + std::chrono::seconds offset; + std::string abbrev; + bool is_dst; +}; + +struct transition +{ + sys_seconds timepoint; + const expanded_ttinfo* info; + + transition(sys_seconds tp, const expanded_ttinfo* i = nullptr) + : timepoint(tp) + , info(i) + {} + + friend + std::ostream& + operator<<(std::ostream& os, const transition& t) + { + using namespace date; + using namespace std::chrono; + using date::operator<<; + os << t.timepoint << "Z "; + if (t.info->offset >= seconds{0}) + os << '+'; + os << make_time(t.info->offset); + if (t.info->is_dst > 0) + os << " daylight "; + else + os << " standard "; + os << t.info->abbrev; + return os; + } +}; + +#endif // USE_OS_TZDB + +} // namespace detail + +} // namespace date +} // namespace util +} // namespace arrow + +#if defined(_MSC_VER) && (_MSC_VER < 1900) +#include "tz.h" +#endif + +#endif // TZ_PRIVATE_H diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index e743b0e041cb8..52784e76ace69 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -44,6 +44,7 @@ add_definitions(-DGANDIVA_BYTE_COMPILE_FILE_PATH="${GANDIVA_BC_OUTPUT_PATH}") set(SRC_FILES annotator.cc bitmap_accumulator.cc + cast_time.cc configuration.cc context_helper.cc decimal_ir.cc @@ -119,6 +120,7 @@ set(GANDIVA_STATIC_TEST_LINK_LIBS gandiva_static ${RE2_LIBRARY} ${ARROW_TEST_LINK_LIBS}) + set(GANDIVA_SHARED_TEST_LINK_LIBS gandiva_shared ${RE2_LIBRARY} diff --git a/cpp/src/gandiva/cast_time.cc b/cpp/src/gandiva/cast_time.cc new file mode 100644 index 0000000000000..ee3fd316eeeee --- /dev/null +++ b/cpp/src/gandiva/cast_time.cc @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/vendored/datetime/tz.h" +#include "gandiva/precompiled/time_fields.h" + +#ifndef GANDIVA_UNIT_TEST +#include "gandiva/exported_funcs.h" +#include "gandiva/gdv_function_stubs.h" + +#include "gandiva/engine.h" + +namespace gandiva { + +void ExportedTimeFunctions::AddMappings(Engine* engine) const { + std::vector args; + auto types = engine->types(); + + // gdv_fn_time_with_zone + args = {types->ptr_type(types->i32_type()), // time fields + types->i8_ptr_type(), // const char* zone + types->i32_type(), // int data_len + types->i64_type()}; // timestamp *ret_time + + engine->AddGlobalMappingForFunc("gdv_fn_time_with_zone", + types->i32_type() /*return_type*/, args, + reinterpret_cast(gdv_fn_time_with_zone)); +} + +} // namespace gandiva +#endif // !GANDIVA_UNIT_TEST + +extern "C" { + +// TODO : Do input validation or make sure the callers do that ? +int gdv_fn_time_with_zone(int* time_fields, const char* zone, int zone_len, + int64_t* ret_time) { + using arrow::util::date::day; + using arrow::util::date::local_days; + using arrow::util::date::locate_zone; + using arrow::util::date::month; + using arrow::util::date::time_zone; + using arrow::util::date::year; + using std::chrono::hours; + using std::chrono::milliseconds; + using std::chrono::minutes; + using std::chrono::seconds; + + using gandiva::TimeFields; + try { + const time_zone* tz = locate_zone(std::string(zone, zone_len)); + *ret_time = tz->to_sys(local_days(year(time_fields[TimeFields::kYear]) / + month(time_fields[TimeFields::kMonth]) / + day(time_fields[TimeFields::kDay])) + + hours(time_fields[TimeFields::kHours]) + + minutes(time_fields[TimeFields::kMinutes]) + + seconds(time_fields[TimeFields::kSeconds]) + + milliseconds(time_fields[TimeFields::kSubSeconds])) + .time_since_epoch() + .count(); + } catch (...) { + return EINVAL; + } + + return 0; +} + +} // extern "C" diff --git a/cpp/src/gandiva/exported_funcs.h b/cpp/src/gandiva/exported_funcs.h index 0ca28c2b5b188..4e028be6ec1d3 100644 --- a/cpp/src/gandiva/exported_funcs.h +++ b/cpp/src/gandiva/exported_funcs.h @@ -45,6 +45,12 @@ class ExportedContextFunctions : public ExportedFuncsBase { }; REGISTER_EXPORTED_FUNCS(ExportedContextFunctions); +// Class for exporting Context functions +class ExportedTimeFunctions : public ExportedFuncsBase { + void AddMappings(Engine* engine) const override; +}; +REGISTER_EXPORTED_FUNCS(ExportedTimeFunctions); + } // namespace gandiva #endif // GANDIVA_EXPORTED_FUNCS_H diff --git a/cpp/src/gandiva/function_registry.cc b/cpp/src/gandiva/function_registry.cc index 452cb6339954c..43eda4dee77d7 100644 --- a/cpp/src/gandiva/function_registry.cc +++ b/cpp/src/gandiva/function_registry.cc @@ -46,7 +46,6 @@ SignatureMap FunctionRegistry::InitPCMap() { auto v1 = GetArithmeticFunctionRegistry(); pc_registry_.insert(std::end(pc_registry_), v1.begin(), v1.end()); - auto v2 = GetDateTimeFunctionRegistry(); pc_registry_.insert(std::end(pc_registry_), v2.begin(), v2.end()); diff --git a/cpp/src/gandiva/function_registry_datetime.cc b/cpp/src/gandiva/function_registry_datetime.cc index 145b7d39395b4..f36e5678c0d73 100644 --- a/cpp/src/gandiva/function_registry_datetime.cc +++ b/cpp/src/gandiva/function_registry_datetime.cc @@ -53,6 +53,10 @@ std::vector GetDateTimeFunctionRegistry() { "castDATE_utf8", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), + NativeFunction("castTIMESTAMP", DataTypeVector{utf8()}, timestamp(), + kResultNullIfNull, "castTIMESTAMP_utf8", + NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), + NativeFunction("to_date", DataTypeVector{utf8(), utf8(), int32()}, date64(), kResultNullInternal, "gdv_fn_to_date_utf8_utf8_int32", NativeFunction::kNeedsContext | diff --git a/cpp/src/gandiva/gdv_function_stubs.h b/cpp/src/gandiva/gdv_function_stubs.h index 154e80b8feecc..8f940cee0f46f 100644 --- a/cpp/src/gandiva/gdv_function_stubs.h +++ b/cpp/src/gandiva/gdv_function_stubs.h @@ -43,6 +43,9 @@ bool in_expr_lookup_int32(int64_t ptr, int32_t value, bool in_validity); bool in_expr_lookup_int64(int64_t ptr, int64_t value, bool in_validity); bool in_expr_lookup_utf8(int64_t ptr, const char* data, int data_len, bool in_validity); + +int gdv_fn_time_with_zone(int* time_fields, const char* zone, int zone_len, + int64_t* ret_time); } #endif // GDV_FUNCTION_STUBS_H diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index 2b629c7a9e4f2..83183bc460dc8 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -58,7 +58,6 @@ add_custom_command( add_custom_target(precompiled ALL DEPENDS ${GANDIVA_BC_OUTPUT_PATH}) -# Add a unittest executable for a precompiled file (used to generate IR) function(add_precompiled_unit_test REL_TEST_NAME) get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE) @@ -79,7 +78,7 @@ endfunction(add_precompiled_unit_test REL_TEST_NAME) if (ARROW_BUILD_TESTS) add_precompiled_unit_test(bitmap_test.cc bitmap.cc) add_precompiled_unit_test(epoch_time_point_test.cc) - add_precompiled_unit_test(time_test.cc time.cc timestamp_arithmetic.cc ../context_helper.cc) + add_precompiled_unit_test(time_test.cc time.cc timestamp_arithmetic.cc ../context_helper.cc ../cast_time.cc ../../arrow/vendored/datetime/tz.cpp) add_precompiled_unit_test(hash_test.cc hash.cc) add_precompiled_unit_test(string_ops_test.cc string_ops.cc ../context_helper.cc) add_precompiled_unit_test(arithmetic_ops_test.cc arithmetic_ops.cc ../context_helper.cc) diff --git a/cpp/src/gandiva/precompiled/epoch_time_point.h b/cpp/src/gandiva/precompiled/epoch_time_point.h index 115f019525118..32d6cea731938 100644 --- a/cpp/src/gandiva/precompiled/epoch_time_point.h +++ b/cpp/src/gandiva/precompiled/epoch_time_point.h @@ -19,7 +19,7 @@ #define GANDIVA_EPOCH_TIME_POINT_H // TODO(wesm): IR compilation does not have any include directories set -#include "../../arrow/vendored/date.h" +#include "../../arrow/vendored/datetime/date.h" // A point of time measured in millis since epoch. class EpochTimePoint { @@ -35,16 +35,18 @@ class EpochTimePoint { int TmMon() const { return static_cast(YearMonthDay().month()) - 1; } int TmYday() const { - auto to_days = date::floor(tp_); - auto first_day_in_year = date::sys_days{YearMonthDay().year() / date::jan / 1}; + auto to_days = arrow::util::date::floor(tp_); + auto first_day_in_year = + arrow::util::date::sys_days{YearMonthDay().year() / arrow::util::date::jan / 1}; return (to_days - first_day_in_year).count(); } int TmMday() const { return static_cast(YearMonthDay().day()); } int TmWday() const { - auto to_days = date::floor(tp_); - return (date::weekday{to_days} - date::Sunday).count(); // NOLINT + auto to_days = arrow::util::date::floor(tp_); + return (arrow::util::date::weekday{to_days} - arrow::util::date::Sunday) // NOLINT + .count(); // NOLINT } int TmHour() const { return static_cast(TimeOfDay().hours().count()); } @@ -57,22 +59,22 @@ class EpochTimePoint { } EpochTimePoint AddYears(int num_years) const { - auto ymd = YearMonthDay() + date::years(num_years); - return EpochTimePoint((date::sys_days{ymd} + // NOLINT + auto ymd = YearMonthDay() + arrow::util::date::years(num_years); + return EpochTimePoint((arrow::util::date::sys_days{ymd} + // NOLINT TimeOfDay().to_duration()) .time_since_epoch()); } EpochTimePoint AddMonths(int num_months) const { - auto ymd = YearMonthDay() + date::months(num_months); - return EpochTimePoint((date::sys_days{ymd} + // NOLINT + auto ymd = YearMonthDay() + arrow::util::date::months(num_months); + return EpochTimePoint((arrow::util::date::sys_days{ymd} + // NOLINT TimeOfDay().to_duration()) .time_since_epoch()); } EpochTimePoint AddDays(int num_days) const { - auto days_since_epoch = date::sys_days{YearMonthDay()} // NOLINT - + date::days(num_days); + auto days_since_epoch = arrow::util::date::sys_days{YearMonthDay()} // NOLINT + + arrow::util::date::days(num_days); return EpochTimePoint( (days_since_epoch + TimeOfDay().to_duration()).time_since_epoch()); } @@ -86,13 +88,16 @@ class EpochTimePoint { int64_t MillisSinceEpoch() const { return tp_.time_since_epoch().count(); } private: - date::year_month_day YearMonthDay() const { - return date::year_month_day{date::floor(tp_)}; // NOLINT + arrow::util::date::year_month_day YearMonthDay() const { + return arrow::util::date::year_month_day{ + arrow::util::date::floor(tp_)}; // NOLINT } - date::time_of_day TimeOfDay() const { - auto millis_since_midnight = tp_ - date::floor(tp_); - return date::time_of_day(millis_since_midnight); + arrow::util::date::time_of_day TimeOfDay() const { + auto millis_since_midnight = + tp_ - arrow::util::date::floor(tp_); + return arrow::util::date::time_of_day( + millis_since_midnight); } std::chrono::time_point tp_; diff --git a/cpp/src/gandiva/precompiled/time.cc b/cpp/src/gandiva/precompiled/time.cc index 2ea7cd4eed032..22c7cbf33568f 100644 --- a/cpp/src/gandiva/precompiled/time.cc +++ b/cpp/src/gandiva/precompiled/time.cc @@ -24,6 +24,7 @@ extern "C" { #include #include "./time_constants.h" +#include "./time_fields.h" #include "./types.h" #define MINS_IN_HOUR 60 @@ -518,6 +519,12 @@ void set_error_for_date(int32 length, const char* input, const char* msg, } date64 castDATE_utf8(int64_t context, const char* input, int32 length) { + using arrow::util::date::day; + using arrow::util::date::month; + using arrow::util::date::sys_days; + using arrow::util::date::year; + using arrow::util::date::year_month_day; + using gandiva::TimeFields; // format : 0 is year, 1 is month and 2 is day. int dateFields[3]; int dateIndex = 0, index = 0, value = 0; @@ -546,21 +553,129 @@ date64 castDATE_utf8(int64_t context, const char* input, int32 length) { * If range of two digits is between 70 - 99 then year = 1970 - 1999 * Else if two digits is between 00 - 69 = 2000 - 2069 */ - if (dateFields[0] < 100) { - if (dateFields[0] < 70) { - dateFields[0] += 2000; + if (dateFields[TimeFields::kYear] < 100) { + if (dateFields[TimeFields::kYear] < 70) { + dateFields[TimeFields::kYear] += 2000; } else { - dateFields[0] += 1900; + dateFields[TimeFields::kYear] += 1900; } } - date::year_month_day day = - date::year(dateFields[0]) / date::month(dateFields[1]) / date::day(dateFields[2]); - if (!day.ok()) { + year_month_day date = year(dateFields[TimeFields::kYear]) / + month(dateFields[TimeFields::kMonth]) / + day(dateFields[TimeFields::kDay]); + if (!date.ok()) { set_error_for_date(length, input, msg, context); return 0; } - return std::chrono::time_point_cast(date::sys_days(day)) + return std::chrono::time_point_cast(sys_days(date)) .time_since_epoch() .count(); } + +/* + * Input consists of mandatory and optional fields. + * Mandatory fields are year, month and day. + * Optional fields are time, displacement and zone. + * Format is [ hours:minutes:seconds][.millis][ displacement|zone] + */ +timestamp castTIMESTAMP_utf8(int64_t context, const char* input, int32 length) { + using arrow::util::date::day; + using arrow::util::date::month; + using arrow::util::date::sys_days; + using arrow::util::date::year; + using arrow::util::date::year_month_day; + using gandiva::TimeFields; + using std::chrono::hours; + using std::chrono::milliseconds; + using std::chrono::minutes; + using std::chrono::seconds; + + int ts_fields[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; + boolean add_displacement = true; + boolean encountered_zone = false; + int ts_field_index = TimeFields::kYear, index = 0, value = 0; + while (ts_field_index < TimeFields::kMax && index < length) { + if (isdigit(input[index])) { + value = (value * 10) + (input[index] - '0'); + } else { + ts_fields[ts_field_index] = value; + value = 0; + + switch (input[index]) { + case '.': + case ':': + case ' ': + ts_field_index++; + break; + case '+': + // +08:00, means time zone is 8 hours ahead. Need to substract. + add_displacement = false; + ts_field_index = TimeFields::kDisplacementHours; + break; + case '-': + // Overloaded as date separator and negative displacement. + ts_field_index = (ts_field_index < 3) ? (ts_field_index + 1) + : TimeFields::kDisplacementHours; + break; + default: + encountered_zone = true; + break; + } + } + if (encountered_zone) { + break; + } + index++; + } + + // Store the last value + if (ts_field_index < TimeFields::kMax) { + ts_fields[ts_field_index++] = value; + } + + // adjust the year + if (ts_fields[TimeFields::kYear] < 100) { + if (ts_fields[TimeFields::kYear] < 70) { + ts_fields[TimeFields::kYear] += 2000; + } else { + ts_fields[TimeFields::kYear] += 1900; + } + } + + // handle timezone + if (encountered_zone) { + int err = 0; + timestamp ret_time = 0; + err = gdv_fn_time_with_zone(&ts_fields[0], (input + index), (length - index), + &ret_time); + if (err) { + const char* msg = "Invalid timestamp or unknown zone for timestamp value "; + set_error_for_date(length, input, msg, context); + return 0; + } + return ret_time; + } + + year_month_day date = year(ts_fields[TimeFields::kYear]) / + month(ts_fields[TimeFields::kMonth]) / + day(ts_fields[TimeFields::kDay]); + if (!date.ok()) { + const char* msg = "Not a valid day for timestamp value "; + set_error_for_date(length, input, msg, context); + return 0; + } + + auto date_time = sys_days(date) + hours(ts_fields[TimeFields::kHours]) + + minutes(ts_fields[TimeFields::kMinutes]) + + seconds(ts_fields[TimeFields::kSeconds]) + + milliseconds(ts_fields[TimeFields::kSubSeconds]); + if (ts_fields[TimeFields::kDisplacementHours] || + ts_fields[TimeFields::kDisplacementMinutes]) { + auto displacement_time = hours(ts_fields[TimeFields::kDisplacementHours]) + + minutes(ts_fields[TimeFields::kDisplacementMinutes]); + date_time = (add_displacement) ? (date_time + displacement_time) + : (date_time - displacement_time); + } + return std::chrono::time_point_cast(date_time).time_since_epoch().count(); +} } // extern "C" diff --git a/cpp/src/gandiva/precompiled/time_fields.h b/cpp/src/gandiva/precompiled/time_fields.h new file mode 100644 index 0000000000000..7131d5c8232b8 --- /dev/null +++ b/cpp/src/gandiva/precompiled/time_fields.h @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef GANDIVA_TIME_FIELDS_H +#define GANDIVA_TIME_FIELDS_H + +namespace gandiva { + +enum TimeFields { + kYear, + kMonth, + kDay, + kHours, + kMinutes, + kSeconds, + kSubSeconds, + kDisplacementHours, + kDisplacementMinutes, + kMax +}; + +} // namespace gandiva +#endif // GANDIVA_TIME_FIELDS_H diff --git a/cpp/src/gandiva/precompiled/time_test.cc b/cpp/src/gandiva/precompiled/time_test.cc index b8ee4dc4fbef3..36ba9b308e621 100644 --- a/cpp/src/gandiva/precompiled/time_test.cc +++ b/cpp/src/gandiva/precompiled/time_test.cc @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include - #include +#include #include "../execution_context.h" #include "gandiva/precompiled/types.h" @@ -50,6 +49,55 @@ TEST(TestTime, TestCastDate) { EXPECT_EQ(castDATE_utf8(context_ptr, "71-12-XX", 8), 0); } +TEST(TestTime, TestCastTimestamp) { + ExecutionContext context; + int64_t context_ptr = reinterpret_cast(&context); + + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "1967-12-1", 9), -65836800000); + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "1972-12-1", 9), 92016000000); + EXPECT_EQ(castDATE_utf8(context_ptr, "67-12-1", 7), 3089923200000); + EXPECT_EQ(castDATE_utf8(context_ptr, "67-1-1", 7), 3061065600000); + EXPECT_EQ(castDATE_utf8(context_ptr, "71-1-1", 7), 31536000000); + + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2000-09-23 9:45:30", 18), 969702330000); + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2000-09-23 9:45:30.920", 22), 969702330920); + + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2000-09-23 9:45:30.920 +08:00", 29), + 969673530920); + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2000-09-23 9:45:30.920 -11:45", 29), + 969744630920); + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "65-03-04 00:20:40.920 +00:30", 28), + 3003349840920); + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "1932-05-18 11:30:00.920 +11:30", 30), + -1187308799080); + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "1857-02-11 20:31:40.920 -05:30", 30), + -3562264699080); + + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2000-09-23 9:45:30.920 Canada/Pacific", 37), + 969727530920); + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2012-02-28 23:30:59 Asia/Kolkata", 32), + 1330452059000); + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "1923-10-07 03:03:03 America/New_York", 36), + -1459094217000); + + // error cases + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "20000923", 8), 0); + EXPECT_EQ(context.get_error(), "Not a valid day for timestamp value 20000923"); + context.Reset(); + + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2000-09-2b", 10), 0); + EXPECT_EQ(context.get_error(), + "Invalid timestamp or unknown zone for timestamp value 2000-09-2b"); + context.Reset(); + + EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2000-09-23 9:45:30.920 Unknown/Zone", 35), + 0); + EXPECT_EQ(context.get_error(), + "Invalid timestamp or unknown zone for timestamp value 2000-09-23 " + "9:45:30.920 Unknown/Zone"); + context.Reset(); +} + TEST(TestTime, TestExtractTime) { // 10:20:33 int32 time_as_millis_in_day = 37233000; diff --git a/cpp/src/gandiva/precompiled/types.h b/cpp/src/gandiva/precompiled/types.h index 9c574ba6e3f0a..7f3b0a09b43ce 100644 --- a/cpp/src/gandiva/precompiled/types.h +++ b/cpp/src/gandiva/precompiled/types.h @@ -157,6 +157,8 @@ int32 utf8_length(int64 context, const char* data, int32 data_len); date64 castDATE_utf8(int64_t execution_context, const char* input, int32 length); +timestamp castTIMESTAMP_utf8(int64_t execution_context, const char* input, int32 length); + } // extern "C" #endif // PRECOMPILED_TYPES_H diff --git a/cpp/src/gandiva/to_date_holder.cc b/cpp/src/gandiva/to_date_holder.cc index f73d05fb71600..6e02a6a0d10db 100644 --- a/cpp/src/gandiva/to_date_holder.cc +++ b/cpp/src/gandiva/to_date_holder.cc @@ -18,7 +18,7 @@ #include #include -#include "arrow/vendored/date.h" +#include "arrow/vendored/datetime/date.h" #include "gandiva/date_utils.h" #include "gandiva/execution_context.h" @@ -90,8 +90,9 @@ int64_t ToDateHolder::operator()(ExecutionContext* context, const std::string& d } *out_valid = true; // ignore the time part - date::sys_seconds secs = date::sys_days(date::year(result.tm_year + 1900) / - (result.tm_mon + 1) / result.tm_mday); + arrow::util::date::sys_seconds secs = + arrow::util::date::sys_days(arrow::util::date::year(result.tm_year + 1900) / + (result.tm_mon + 1) / result.tm_mday); int64_t seconds_since_epoch = secs.time_since_epoch().count(); if (seconds_since_epoch == 0) { return_error(context, data); From 4e9db2f1cbe5ae5ec4a3d0b9bfdc080822e8c638 Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Thu, 31 Jan 2019 15:45:18 +0100 Subject: [PATCH 10/20] ARROW-4424: [Python] Install tensorflow and keras-preprocessing in manylinux1 container Author: Korn, Uwe Closes #3531 from xhochy/ARROW-4424 and squashes the following commits: 7359b437 ARROW-4424: Install tensorflow and keras-preprocessing in manylinux1 container --- .travis.yml | 2 +- ci/travis_script_manylinux.sh | 2 +- python/manylinux1/build_arrow.sh | 7 ------- python/manylinux1/scripts/build_virtualenvs.sh | 17 ++++++++++++----- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7abeb99e5207d..26d389244963c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -224,7 +224,7 @@ matrix: - name: "[manylinux1] Python" language: cpp before_script: - - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest; fi + - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-4424; fi script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi - name: "Java w/ OpenJDK 8" diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 588d0f9a7bbb6..a7d82cc76c2bc 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -21,7 +21,7 @@ set -ex pushd python/manylinux1 -docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io -v $PWD/../../:/arrow quay.io/xhochy/arrow_manylinux1_x86_64_base:latest /io/build_arrow.sh +docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io -v $PWD/../../:/arrow quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-4424 /io/build_arrow.sh # Testing for https://issues.apache.org/jira/browse/ARROW-2657 # These tests cannot be run inside of the docker container, since TensorFlow diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 7488d380ef54a..75537a64e732d 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -67,13 +67,6 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do export BUILD_ARROW_GANDIVA=OFF fi - # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 - if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then - if [ $PYTHON != "3.7" ]; then - $PIP install tensorflow==1.11.0 - fi - fi - echo "=== (${PYTHON}) Building Arrow C++ libraries ===" ARROW_BUILD_DIR=/tmp/build-PY${PYTHON}-${U_WIDTH} mkdir -p "${ARROW_BUILD_DIR}" diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh index bc3e73f9629a3..a737e2f6ef968 100755 --- a/python/manylinux1/scripts/build_virtualenvs.sh +++ b/python/manylinux1/scripts/build_virtualenvs.sh @@ -33,15 +33,22 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})" echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ===" - $PIP install "numpy==1.14.5" - $PIP install "cython==0.29.3" - $PIP install "pandas==0.23.4" - $PIP install "virtualenv==15.1.0" + $PIP install "numpy==1.14.5" "cython==0.29.3" "virtualenv==16.3.0" + # Pandas requires numpy and cython + $PIP install "pandas==0.24.0" + + # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 + if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then + if [ $PYTHON != "3.7" ]; then + $PIP install "tensorflow==1.11.0" "Keras-Preprocessing==1.0.5" + fi + fi + echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for tests ===" "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}-${U_WIDTH} source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate - pip install pytest hypothesis 'numpy==1.14.5' 'pandas==0.23.4' + pip install pytest hypothesis 'numpy==1.14.5' 'pandas==0.24.0' deactivate done From 28c9eeb2991a588080346d2550aa955cd37ef693 Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Thu, 31 Jan 2019 14:47:01 +0000 Subject: [PATCH 11/20] include new jest configs to ensure verify release script works (#3532) --- js/package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/js/package.json b/js/package.json index fe37d69c72a42..22a1df9032aa6 100644 --- a/js/package.json +++ b/js/package.json @@ -53,7 +53,9 @@ "tsconfig", "README.md", "gulpfile.js", - "npm-release.sh" + "npm-release.sh", + "jest.config.js", + "jest.coverage.config.js" ], "dependencies": { "@types/flatbuffers": "^1.9.0", From 36e26fe473ab776982a52861bbc6875431abae7c Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 31 Jan 2019 11:04:36 -0600 Subject: [PATCH 12/20] ARROW-4431: [C++] Fixes for gRPC vendored builds This enables Arrow Flight to build gRPC successfully from ExternalProject both with all dependencies being built from EP or from provided packages via _HOME variables. Author: Wes McKinney Closes #3527 from pitrou/grpc-vendored-fixes and squashes the following commits: fb8fdf33c No more verbose build in CI d84b9190e Restore all builds c778bece6 Revert vendored Boost changes 65ac53d63 Revert Boost link order d5b8f547d Try removing SYSTEM 7563955c1 Verbose build again 1880bfa59 Fix Boost include dir dd4f1eb5e debugging 369f0e520 Use verbose makefile for debugging accaeb036 verbose windows build for debugging 318181333 Use BEFORE SYSTEM with vendored Boost headers 0a2f9673b More tweaks / fixed f2db99a74 Change Boost library include order ad994b16d Fix brotli library locations after refactor 73405450e Fix Brotli headers copy step 3140b3c6d Quote thirdparty install prefixes, verbose thirdparty build in MSVC 024a2dca6 Fixes for gRPC vendored builds --- .travis.yml | 1 + ci/cpp-msvc-build-main.bat | 2 + cpp/cmake_modules/ThirdpartyToolchain.cmake | 369 +++++++++++++------- cpp/src/arrow/flight/CMakeLists.txt | 4 +- cpp/src/arrow/flight/perf-server.cc | 2 + cpp/src/arrow/memory_pool.cc | 2 +- cpp/thirdparty/versions.txt | 4 +- 7 files changed, 254 insertions(+), 130 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26d389244963c..c037cd5d914e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -188,6 +188,7 @@ matrix: - ARROW_TRAVIS_GANDIVA=1 - ARROW_TRAVIS_GANDIVA_JAVA=1 - ARROW_TRAVIS_OPTIONAL_INSTALL=1 + - ARROW_TRAVIS_VERBOSE=0 - ARROW_BUILD_WARNING_LEVEL=CHECKIN # ARROW-3803: The Xcode 8.3 image has Boost libraries in /usr/local/lib # which can get loaded before the toolchain Boost libraries. These seem to diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat index c36c6bd5c53d9..ccd64e3d33334 100644 --- a/ci/cpp-msvc-build-main.bat +++ b/ci/cpp-msvc-build-main.bat @@ -51,6 +51,8 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^ -DARROW_BUILD_STATIC=OFF ^ -DARROW_BUILD_TESTS=ON ^ -DARROW_BUILD_EXAMPLES=ON ^ + -DARROW_BUILD_EXAMPLES=ON ^ + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ -DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^ -DCMAKE_CXX_FLAGS_RELEASE="/MD %CMAKE_CXX_FLAGS_RELEASE%" ^ -DARROW_PARQUET=ON ^ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 606b476992540..ce5073f6d076d 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -17,6 +17,8 @@ add_custom_target(toolchain) +set(THIRDPARTY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_thirdparty") + # ---------------------------------------------------------------------- # Toolchain linkage options @@ -151,6 +153,18 @@ else() set(ARROW_WITH_THRIFT OFF) endif() +if (ARROW_FLIGHT) + set(ARROW_WITH_GRPC ON) +endif() + +if (ARROW_FLIGHT OR ARROW_IPC) + set(ARROW_WITH_RAPIDJSON ON) +endif() + +if (ARROW_ORC OR ARROW_FLIGHT OR ARROW_GANDIVA) + set(ARROW_WITH_PROTOBUF ON) +endif() + # ---------------------------------------------------------------------- # Versions and URLs for toolchain builds, which also can be used to configure # offline builds @@ -191,6 +205,12 @@ else() set(BROTLI_SOURCE_URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz") endif() +if (DEFINED ENV{ARROW_CARES_URL}) + set(CARES_SOURCE_URL "$ENV{ARROW_CARES_URL}") +else() + set(CARES_SOURCE_URL "https://c-ares.haxx.se/download/c-ares-${CARES_VERSION}.tar.gz") +endif() + if (DEFINED ENV{ARROW_DOUBLE_CONVERSION_URL}) set(DOUBLE_CONVERSION_SOURCE_URL "$ENV{ARROW_DOUBLE_CONVERSION_URL}") else() @@ -487,7 +507,7 @@ else() endif() endif() -message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS}) +message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) if (NOT ARROW_BOOST_HEADER_ONLY) @@ -512,17 +532,17 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIR}) # Google double-conversion if("${DOUBLE_CONVERSION_HOME}" STREQUAL "") - set(DOUBLE_CONVERSION_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/double-conversion_ep/src/double-conversion_ep") - set(DOUBLE_CONVERSION_HOME "${DOUBLE_CONVERSION_PREFIX}") - set(DOUBLE_CONVERSION_INCLUDE_DIR "${DOUBLE_CONVERSION_PREFIX}/include") - set(DOUBLE_CONVERSION_STATIC_LIB "${DOUBLE_CONVERSION_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}double-conversion${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(DOUBLE_CONVERSION_HOME "${THIRDPARTY_PREFIX}") + set(DOUBLE_CONVERSION_INCLUDE_DIR "${THIRDPARTY_PREFIX}/include") + set(DOUBLE_CONVERSION_STATIC_LIB "${THIRDPARTY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}double-conversion${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(DOUBLE_CONVERSION_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${DOUBLE_CONVERSION_PREFIX}) + set(DOUBLE_CONVERSION_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${THIRDPARTY_PREFIX}") ExternalProject_Add(double-conversion_ep ${EP_LOG_OPTIONS} - INSTALL_DIR ${DOUBLE_CONVERSION_PREFIX} + INSTALL_DIR ${THIRDPARTY_PREFIX} URL ${DOUBLE_CONVERSION_SOURCE_URL} CMAKE_ARGS ${DOUBLE_CONVERSION_CMAKE_ARGS} BUILD_BYPRODUCTS "${DOUBLE_CONVERSION_STATIC_LIB}") @@ -552,12 +572,20 @@ message(STATUS "double-conversion static library: ${DOUBLE_CONVERSION_STATIC_LIB # ---------------------------------------------------------------------- # gflags -if(ARROW_BUILD_TESTS OR - ARROW_BUILD_BENCHMARKS OR - (ARROW_USE_GLOG AND GLOG_HOME)) +if (ARROW_BUILD_TESTS OR + ARROW_BUILD_BENCHMARKS OR + (ARROW_USE_GLOG AND GLOG_HOME) OR + (ARROW_WITH_GRPC AND NOT GRPC_HOME)) + set(ARROW_NEED_GFLAGS 1) +else() + set(ARROW_NEED_GFLAGS 0) +endif() + +if(ARROW_NEED_GFLAGS) # gflags (formerly Googleflags) command line parsing if("${GFLAGS_HOME}" STREQUAL "") - set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep") + set(GFLAGS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) + set(GFLAGS_PREFIX "${THIRDPARTY_PREFIX}") set(GFLAGS_HOME "${GFLAGS_PREFIX}") set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include") if(MSVC) @@ -567,7 +595,7 @@ if(ARROW_BUILD_TESTS OR endif() set(GFLAGS_VENDORED 1) set(GFLAGS_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX} + "-DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}" -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON -DBUILD_PACKAGING=OFF @@ -615,7 +643,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) -Wno-ignored-attributes) endif() - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") + set(GTEST_PREFIX "${THIRDPARTY_PREFIX}") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GTEST_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -623,8 +651,8 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GTEST_VENDORED 1) set(GTEST_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX} - -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) + "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" + -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) if (MSVC AND NOT ARROW_USE_STATIC_CRT) set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON) endif() @@ -679,14 +707,14 @@ if(ARROW_BUILD_BENCHMARKS) set(GBENCHMARK_CMAKE_CXX_FLAGS "${GBENCHMARK_CMAKE_CXX_FLAGS} -stdlib=libc++") endif() - set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install") + set(GBENCHMARK_PREFIX "${THIRDPARTY_PREFIX}") set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include") set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GBENCHMARK_VENDORED 1) set(GBENCHMARK_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${GBENCHMARK_PREFIX} - -DBENCHMARK_ENABLE_TESTING=OFF - -DCMAKE_CXX_FLAGS=${GBENCHMARK_CMAKE_CXX_FLAGS}) + "-DCMAKE_INSTALL_PREFIX=${GBENCHMARK_PREFIX}" + -DBENCHMARK_ENABLE_TESTING=OFF + -DCMAKE_CXX_FLAGS=${GBENCHMARK_CMAKE_CXX_FLAGS}) if (APPLE) set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON") endif() @@ -712,21 +740,24 @@ if(ARROW_BUILD_BENCHMARKS) endif() endif() -if (ARROW_IPC) +if (ARROW_WITH_RAPIDJSON) # RapidJSON, header only dependency if("${RAPIDJSON_HOME}" STREQUAL "") + set(RAPIDJSON_HOME "${THIRDPARTY_PREFIX}") + set(RAPIDJSON_CMAKE_ARGS + -DRAPIDJSON_BUILD_DOC=OFF + -DRAPIDJSON_BUILD_EXAMPLES=OFF + -DRAPIDJSON_BUILD_TESTS=OFF + "-DCMAKE_INSTALL_PREFIX=${THIRDPARTY_PREFIX}") + ExternalProject_Add(rapidjson_ep + ${EP_LOG_OPTIONS} PREFIX "${CMAKE_BINARY_DIR}" URL ${RAPIDJSON_SOURCE_URL} URL_MD5 ${RAPIDJSON_SOURCE_MD5} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - BUILD_IN_SOURCE 1 - ${EP_LOG_OPTIONS} - INSTALL_COMMAND "") + CMAKE_ARGS ${RAPIDJSON_CMAKE_ARGS}) - ExternalProject_Get_Property(rapidjson_ep SOURCE_DIR) - set(RAPIDJSON_INCLUDE_DIR "${SOURCE_DIR}/include") + set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_HOME}/include") set(RAPIDJSON_VENDORED 1) add_dependencies(toolchain rapidjson_ep) else() @@ -738,8 +769,9 @@ if (ARROW_IPC) ## Flatbuffers if("${FLATBUFFERS_HOME}" STREQUAL "") - set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install") set(FLATBUFFERS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) + set(FLATBUFFERS_PREFIX "${THIRDPARTY_PREFIX}") + if (MSVC) set(FLATBUFFERS_CMAKE_CXX_FLAGS "/EHsc") endif() @@ -747,10 +779,10 @@ if (ARROW_IPC) # RELEASE build is required for `flatc` to be installed. set(FLATBUFFERS_BUILD_TYPE RELEASE) set(FLATBUFFERS_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_BUILD_TYPE=${FLATBUFFERS_BUILD_TYPE} - -DCMAKE_INSTALL_PREFIX=${FLATBUFFERS_PREFIX} - -DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS} - -DFLATBUFFERS_BUILD_TESTS=OFF) + -DCMAKE_BUILD_TYPE=${FLATBUFFERS_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${FLATBUFFERS_PREFIX} + -DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS} + -DFLATBUFFERS_BUILD_TESTS=OFF) ExternalProject_Add(flatbuffers_ep URL ${FLATBUFFERS_SOURCE_URL} @@ -785,7 +817,7 @@ if (ARROW_JEMALLOC) # find_package(jemalloc) set(ARROW_JEMALLOC_USE_SHARED OFF) - set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/") + set(JEMALLOC_PREFIX "${THIRDPARTY_PREFIX}") set(JEMALLOC_HOME "${JEMALLOC_PREFIX}") set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include") set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}") @@ -804,7 +836,8 @@ if (ARROW_JEMALLOC) # Don't use the include directory directly so that we can point to a path # that is unique to our codebase. - include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/") + include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}") + ADD_THIRDPARTY_LIB(jemalloc STATIC_LIB ${JEMALLOC_STATIC_LIB} SHARED_LIB ${JEMALLOC_SHARED_LIB} @@ -865,7 +898,7 @@ if (ARROW_WITH_ZLIB) ADD_THIRDPARTY_LIB(zlib SHARED_LIB ${ZLIB_SHARED_LIB}) set(ZLIB_LIBRARY zlib_shared) else() - set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install") + set(ZLIB_PREFIX "${THIRDPARTY_PREFIX}") set(ZLIB_HOME "${ZLIB_PREFIX}") set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include") if (MSVC) @@ -879,8 +912,8 @@ if (ARROW_WITH_ZLIB) endif() set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX} - -DBUILD_SHARED_LIBS=OFF) + "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" + -DBUILD_SHARED_LIBS=OFF) ADD_THIRDPARTY_LIB(zlib STATIC_LIB ${ZLIB_STATIC_LIB}) set(ZLIB_LIBRARY zlib_static) @@ -901,7 +934,7 @@ if (ARROW_WITH_SNAPPY) # Snappy if("${SNAPPY_HOME}" STREQUAL "") - set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep/src/snappy_ep-install") + set(SNAPPY_PREFIX "${THIRDPARTY_PREFIX}") set(SNAPPY_HOME "${SNAPPY_PREFIX}") set(SNAPPY_INCLUDE_DIR "${SNAPPY_PREFIX}/include") if (MSVC) @@ -921,9 +954,9 @@ if (ARROW_WITH_SNAPPY) if (WIN32) set(SNAPPY_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_AR=${CMAKE_AR} - -DCMAKE_RANLIB=${CMAKE_RANLIB} - -DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}) + -DCMAKE_AR=${CMAKE_AR} + -DCMAKE_RANLIB=${CMAKE_RANLIB} + "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}") set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt ./CMakeLists.txt && @@ -969,7 +1002,7 @@ if (ARROW_WITH_BROTLI) # Brotli if("${BROTLI_HOME}" STREQUAL "") - set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install") + set(BROTLI_PREFIX "${THIRDPARTY_PREFIX}") set(BROTLI_HOME "${BROTLI_PREFIX}") set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include") if (MSVC) @@ -977,13 +1010,13 @@ if (ARROW_WITH_BROTLI) else() set(BROTLI_LIB_DIR lib) endif() - set(BROTLI_STATIC_LIBRARY_ENC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BROTLI_STATIC_LIBRARY_DEC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BROTLI_STATIC_LIBRARY_ENC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BROTLI_STATIC_LIBRARY_DEC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}") set(BROTLI_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX} - -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE} - -DBUILD_SHARED_LIBS=OFF) + "-DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=lib + -DBUILD_SHARED_LIBS=OFF) ExternalProject_Add(brotli_ep URL ${BROTLI_SOURCE_URL} @@ -996,7 +1029,7 @@ if (ARROW_WITH_BROTLI) ExternalProject_Get_Property(brotli_ep SOURCE_DIR) ExternalProject_Add_Step(brotli_ep headers_copy - COMMAND xcopy /E /I include ..\\..\\..\\brotli_ep\\src\\brotli_ep-install\\include /Y + COMMAND xcopy /E /I include ..\\..\\..\\arrow_thirdparty\\include /Y DEPENDEES build WORKING_DIRECTORY ${SOURCE_DIR}) endif() @@ -1094,16 +1127,16 @@ if (ARROW_WITH_ZSTD) # ZSTD if("${ZSTD_HOME}" STREQUAL "") - set(ZSTD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-install") + set(ZSTD_PREFIX "${THIRDPARTY_PREFIX}") set(ZSTD_INCLUDE_DIR "${ZSTD_PREFIX}/include") set(ZSTD_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX} - -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} - -DZSTD_BUILD_PROGRAMS=off - -DZSTD_BUILD_SHARED=off - -DZSTD_BUILD_STATIC=on - -DZSTD_MULTITHREAD_SUPPORT=off) + "-DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} + -DZSTD_BUILD_PROGRAMS=off + -DZSTD_BUILD_SHARED=off + -DZSTD_BUILD_STATIC=on + -DZSTD_MULTITHREAD_SUPPORT=off) if (MSVC) set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/${CMAKE_INSTALL_LIBDIR}/zstd_static.lib") @@ -1154,13 +1187,13 @@ endif() if (ARROW_GANDIVA) # re2 if ("${RE2_HOME}" STREQUAL "") - set (RE2_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/re2_ep-install") + set (RE2_PREFIX "${THIRDPARTY_PREFIX}") set (RE2_HOME "${RE2_PREFIX}") set (RE2_INCLUDE_DIR "${RE2_PREFIX}/include") set (RE2_STATIC_LIB "${RE2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}re2${CMAKE_STATIC_LIBRARY_SUFFIX}") set(RE2_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${RE2_PREFIX}) + "-DCMAKE_INSTALL_PREFIX=${RE2_PREFIX}") ExternalProject_Add(re2_ep ${EP_LOG_OPTIONS} @@ -1192,10 +1225,10 @@ endif () # ---------------------------------------------------------------------- # Protocol Buffers (required for ORC and Flight and Gandiva libraries) -if (ARROW_ORC OR ARROW_FLIGHT OR ARROW_GANDIVA) +if (ARROW_WITH_PROTOBUF) # protobuf if ("${PROTOBUF_HOME}" STREQUAL "") - set (PROTOBUF_PREFIX "${THIRDPARTY_DIR}/protobuf_ep-install") + set (PROTOBUF_PREFIX "${THIRDPARTY_PREFIX}") set (PROTOBUF_HOME "${PROTOBUF_PREFIX}") set (PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include") set (PROTOBUF_STATIC_LIB "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -1240,53 +1273,135 @@ endif() # ---------------------------------------------------------------------- # Dependencies for Arrow Flight RPC -if (ARROW_FLIGHT) +if (ARROW_WITH_GRPC) + if ("${CARES_HOME}" STREQUAL "") + set(CARES_VENDORED 1) + set(CARES_PREFIX "${THIRDPARTY_PREFIX}") + set(CARES_HOME "${CARES_PREFIX}") + set(CARES_INCLUDE_DIR "${CARES_PREFIX}/include") + + # If you set -DCARES_SHARED=ON then the build system names the library + # libcares_static.a + set(CARES_STATIC_LIB "${CARES_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}cares${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(CARES_CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCARES_STATIC=ON + -DCARES_SHARED=OFF + "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" + "-DCMAKE_INSTALL_PREFIX=${CARES_PREFIX}") + + ExternalProject_Add(cares_ep + ${EP_LOG_OPTIONS} + URL ${CARES_SOURCE_URL} + CMAKE_ARGS ${CARES_CMAKE_ARGS} + BUILD_BYPRODUCTS "${CARES_STATIC_LIB}") + else() + set(CARES_VENDORED 0) + find_package(c-ares REQUIRED + PATHS ${CARES_HOME} + NO_DEFAULT_PATH) + if(TARGET c-ares::cares) + get_property(CARES_STATIC_LIB TARGET c-ares::cares_static PROPERTY LOCATION) + endif() + endif() + message(STATUS "c-ares library: ${CARES_STATIC_LIB}") + + add_custom_target(grpc) + if ("${GRPC_HOME}" STREQUAL "") set(GRPC_VENDORED 1) set(GRPC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep-build") - set(GRPC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep/src/grpc_ep-install") + set(GRPC_PREFIX "${THIRDPARTY_PREFIX}") set(GRPC_HOME "${GRPC_PREFIX}") set(GRPC_INCLUDE_DIR "${GRPC_PREFIX}/include") - set(GRPC_STATIC_LIBRARY_GPR "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gpr${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_STATIC_LIBRARY_GRPC "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpc${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_STATIC_LIBRARY_GRPCPP "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_STATIC_LIBRARY_ADDRESS_SORTING "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}address_sorting${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_STATIC_LIBRARY_CARES "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/third_party/cares/cares/lib/${CMAKE_STATIC_LIBRARY_PREFIX}cares${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GRPC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} - -DBUILD_SHARED_LIBS=OFF) + "-DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX}" + -DBUILD_SHARED_LIBS=OFF) + + set(GRPC_STATIC_LIBRARY_GPR "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gpr${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GRPC_STATIC_LIBRARY_GRPC "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}grpc${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GRPC_STATIC_LIBRARY_GRPCPP "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GRPC_STATIC_LIBRARY_ADDRESS_SORTING "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}address_sorting${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin") + + set(GRPC_CMAKE_PREFIX "${THIRDPARTY_PREFIX}") + + add_custom_target(grpc_dependencies) + + if (CARES_VENDORED) + add_dependencies(grpc_dependencies cares_ep) + else() + set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${CARES_HOME}") + endif() + + if (GFLAGS_VENDORED) + add_dependencies(grpc_dependencies gflags_ep) + else() + set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${GFLAGS_HOME}") + endif() + + if (PROTOBUF_VENDORED) + add_dependencies(grpc_dependencies protobuf_ep) + else() + set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${PROTOBUF_HOME}") + endif() + + # ZLIB is never vendored + if(NOT "${ZLIB_HOME}" STREQUAL "") + set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${ZLIB_HOME}") + endif() + + if (RAPIDJSON_VENDORED) + add_dependencies(grpc_dependencies rapidjson_ep) + endif() + # Yuck, see https://stackoverflow.com/a/45433229/776560 + string(REPLACE ";" "|" GRPC_PREFIX_PATH_ALT_SEP "${GRPC_CMAKE_PREFIX}") + + set(GRPC_CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_PREFIX_PATH="${GRPC_PREFIX_PATH_ALT_SEP}" + "-DgRPC_CARES_PROVIDER=package" + "-DgRPC_GFLAGS_PROVIDER=package" + "-DgRPC_PROTOBUF_PROVIDER=package" + "-DgRPC_SSL_PROVIDER=package" + "-DgRPC_ZLIB_PROVIDER=package" + "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" + "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" + "-DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=lib + -DBUILD_SHARED_LIBS=OFF) + + # XXX the gRPC git checkout is huge and takes a long time + # Ideally, we should be able to use the tarballs, but they don't contain + # vendored dependencies such as c-ares... ExternalProject_Add(grpc_ep - GIT_REPOSITORY "https://github.com/grpc/grpc" - GIT_TAG ${GRPC_VERSION} - BUILD_BYPRODUCTS "${GRPC_STATIC_LIBRARY_GPR}" "${GRPC_STATIC_LIBRARY_GRPC}" "${GRPC_STATIC_LIBRARY_GRPCPP}" - ${GRPC_BUILD_BYPRODUCTS} - ${EP_LOG_OPTIONS} + URL ${GRPC_SOURCE_URL} + LIST_SEPARATOR | + BUILD_BYPRODUCTS + ${GRPC_STATIC_LIBRARY_GPR} + ${GRPC_STATIC_LIBRARY_GRPC} + ${GRPC_STATIC_LIBRARY_GRPCPP} + ${GRPC_STATIC_LIBRARY_ADDRESS_SORTING} + ${GRPC_CPP_PLUGIN} CMAKE_ARGS ${GRPC_CMAKE_ARGS} ${EP_LOG_OPTIONS}) + add_dependencies(grpc_ep grpc_dependencies) + set(GPR_STATIC_LIB "${GRPC_STATIC_LIBRARY_GPR}") set(GRPC_STATIC_LIB "${GRPC_STATIC_LIBRARY_GRPC}") set(GRPCPP_STATIC_LIB "${GRPC_STATIC_LIBRARY_GRPCPP}") set(GRPC_ADDRESS_SORTING_STATIC_LIB "${GRPC_STATIC_LIBRARY_ADDRESS_SORTING}") - # XXX(wesm): relying on vendored c-ares provided by gRPC for the time being - set(CARES_STATIC_LIB "${GRPC_STATIC_LIBRARY_CARES}") - set(GRPC_CPP_PLUGIN "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/grpc_cpp_plugin") + + add_dependencies(grpc grpc_ep) + add_dependencies(toolchain grpc) else() find_package(gRPC REQUIRED) set(GRPC_VENDORED 0) endif() - # If we built gRPC ourselves, we should use its c-ares. - if ("${CARES_STATIC_LIB}" STREQUAL "") - if (NOT "${CARES_HOME}" STREQUAL "") - set(CARES_STATIC_LIB "${CARES_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}cares_static${CMAKE_STATIC_LIBRARY_SUFFIX}") - elseif (c-ares_FOUND) - get_property(CARES_STATIC_LIB TARGET c-ares::cares_static PROPERTY LOCATION) - endif() - endif() - message(STATUS "Found the c-ares library: ${CARES_STATIC_LIB}") - if ("${GRPC_CPP_PLUGIN}" STREQUAL "") message(SEND_ERROR "Please set GRPC_CPP_PLUGIN.") endif() @@ -1315,7 +1430,7 @@ endif() if (ARROW_ORC) # orc if ("${ORC_HOME}" STREQUAL "") - set(ORC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/orc_ep-install") + set(ORC_PREFIX "${THIRDPARTY_PREFIX}") set(ORC_HOME "${ORC_PREFIX}") set(ORC_INCLUDE_DIR "${ORC_PREFIX}/include") set(ORC_STATIC_LIB "${ORC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}orc${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -1333,18 +1448,18 @@ if (ARROW_ORC) # ${LZ4_HOME}/include, which forces us to specify the include directory # manually as well. set (ORC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${ORC_PREFIX} - -DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS} - -DBUILD_LIBHDFSPP=OFF - -DBUILD_JAVA=OFF - -DBUILD_TOOLS=OFF - -DBUILD_CPP_TESTS=OFF - -DINSTALL_VENDORED_LIBS=OFF - -DPROTOBUF_HOME=${PROTOBUF_HOME} - -DLZ4_HOME=${LZ4_HOME} - -DLZ4_INCLUDE_DIR=${LZ4_INCLUDE_DIR} - -DSNAPPY_HOME=${SNAPPY_HOME} - -DZLIB_HOME=${ZLIB_HOME}) + "-DCMAKE_INSTALL_PREFIX=${ORC_PREFIX}" + -DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS} + -DBUILD_LIBHDFSPP=OFF + -DBUILD_JAVA=OFF + -DBUILD_TOOLS=OFF + -DBUILD_CPP_TESTS=OFF + -DINSTALL_VENDORED_LIBS=OFF + -DPROTOBUF_HOME=${PROTOBUF_HOME} + -DLZ4_HOME=${LZ4_HOME} + -DLZ4_INCLUDE_DIR=${LZ4_INCLUDE_DIR} + -DSNAPPY_HOME=${SNAPPY_HOME} + -DZLIB_HOME=${ZLIB_HOME}) ExternalProject_Add(orc_ep URL ${ORC_SOURCE_URL} @@ -1390,25 +1505,25 @@ if (ARROW_WITH_THRIFT) find_package(Thrift) if (NOT THRIFT_FOUND) - set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep/src/thrift_ep-install") + set(THRIFT_PREFIX "${THIRDPARTY_PREFIX}") set(THRIFT_HOME "${THRIFT_PREFIX}") set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include") set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift") set(THRIFT_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX} - -DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib - -DBUILD_SHARED_LIBS=OFF - -DBUILD_TESTING=OFF - -DBUILD_EXAMPLES=OFF - -DBUILD_TUTORIALS=OFF - -DWITH_QT4=OFF - -DWITH_C_GLIB=OFF - -DWITH_JAVA=OFF - -DWITH_PYTHON=OFF - -DWITH_HASKELL=OFF - -DWITH_CPP=ON - -DWITH_STATIC_LIB=ON - -DWITH_LIBEVENT=OFF) + "-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}" + -DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DBUILD_EXAMPLES=OFF + -DBUILD_TUTORIALS=OFF + -DWITH_QT4=OFF + -DWITH_C_GLIB=OFF + -DWITH_JAVA=OFF + -DWITH_PYTHON=OFF + -DWITH_HASKELL=OFF + -DWITH_CPP=ON + -DWITH_STATIC_LIB=ON + -DWITH_LIBEVENT=OFF) # Thrift also uses boost. Forward important boost settings if there were ones passed. if (DEFINED BOOST_ROOT) @@ -1444,7 +1559,7 @@ if (NOT THRIFT_FOUND) if (MSVC) set(WINFLEXBISON_VERSION 2.4.9) - set(WINFLEXBISON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install") + set(WINFLEXBISON_PREFIX "${THIRDPARTY_PREFIX}") ExternalProject_Add(winflexbison_ep URL https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip URL_HASH MD5=a2e979ea9928fbf8567e995e9c0df765 @@ -1524,9 +1639,9 @@ endif() # ARROW_HIVESERVER2 if (ARROW_USE_GLOG) if("${GLOG_HOME}" STREQUAL "") - set(GLOG_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/glog_ep-prefix/src/glog_ep") - set(GLOG_INCLUDE_DIR "${GLOG_BUILD_DIR}/include") - set(GLOG_STATIC_LIB "${GLOG_BUILD_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}glog${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GLOG_PREFIX "${THIRDPARTY_PREFIX}") + set(GLOG_INCLUDE_DIR "${GLOG_PREFIX}/include") + set(GLOG_STATIC_LIB "${GLOG_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}glog${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC") if (Threads::Threads) @@ -1542,13 +1657,13 @@ if (ARROW_USE_GLOG) endif() set(GLOG_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${GLOG_BUILD_DIR} - -DBUILD_SHARED_LIBS=OFF - -DBUILD_TESTING=OFF - -DWITH_GFLAGS=OFF - -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_CXX_FLAGS} - -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_C_FLAGS} - -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS}) + "-DCMAKE_INSTALL_PREFIX=${GLOG_PREFIX}" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DWITH_GFLAGS=OFF + -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_CXX_FLAGS} + -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_C_FLAGS} + -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS}) message(STATUS "Glog version: ${GLOG_VERSION}") ExternalProject_Add(glog_ep URL ${GLOG_SOURCE_URL} diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index f59ea3c5e6757..b8b4d8d336365 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -45,7 +45,7 @@ set(FLIGHT_GENERATED_PROTO_FILES "${CMAKE_CURRENT_BINARY_DIR}/Flight.grpc.pb.cc" "${CMAKE_CURRENT_BINARY_DIR}/Flight.grpc.pb.h") -set(PROTO_DEPENDS ${FLIGHT_PROTO} ${PROTOBUF_LIBRARY}) +set(PROTO_DEPENDS ${FLIGHT_PROTO} ${PROTOBUF_LIBRARY} grpc) add_custom_command( OUTPUT ${FLIGHT_GENERATED_PROTO_FILES} @@ -127,6 +127,7 @@ if (ARROW_BUILD_BENCHMARKS) perf.pb.cc) target_link_libraries(flight-perf-server arrow_flight_static + arrow_testing_static ${ARROW_FLIGHT_STATIC_LINK_LIBS} gflags_static ${GTEST_LIBRARY}) @@ -136,6 +137,7 @@ if (ARROW_BUILD_BENCHMARKS) perf.pb.cc) target_link_libraries(flight-benchmark arrow_flight_static + arrow_testing_static ${ARROW_FLIGHT_STATIC_LINK_LIBS} gflags_static ${GTEST_LIBRARY}) diff --git a/cpp/src/arrow/flight/perf-server.cc b/cpp/src/arrow/flight/perf-server.cc index ce2ec7bca6cff..add544276f529 100644 --- a/cpp/src/arrow/flight/perf-server.cc +++ b/cpp/src/arrow/flight/perf-server.cc @@ -69,6 +69,8 @@ class PerfDataStream : public FlightDataStream { batch_ = RecordBatch::Make(schema, batch_length_, arrays_); } + std::shared_ptr schema() override { return schema_; } + Status Next(IpcPayload* payload) override { if (records_sent_ >= total_records_) { // Signal that iteration is over diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc index 3e0366a19da41..103771bf527a7 100644 --- a/cpp/src/arrow/memory_pool.cc +++ b/cpp/src/arrow/memory_pool.cc @@ -32,7 +32,7 @@ // Needed to support jemalloc 3 and 4 #define JEMALLOC_MANGLE // Explicitly link to our version of jemalloc -#include "jemalloc_ep/dist/include/jemalloc/jemalloc.h" +#include "arrow_thirdparty/include/jemalloc/jemalloc.h" #endif namespace arrow { diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index a2393b6fb3eb0..00cd31e6df969 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,12 +25,13 @@ BOOST_VERSION=1.67.0 BROTLI_VERSION=v0.6.0 +CARES_VERSION=1.15.0 DOUBLE_CONVERSION_VERSION=v3.1.1 FLATBUFFERS_VERSION=02a7807dd8d26f5668ffbbec0360dc107bbfabd5 GBENCHMARK_VERSION=v1.4.1 GFLAGS_VERSION=v2.2.0 GLOG_VERSION=v0.3.5 -GRPC_VERSION=v1.14.1 +GRPC_VERSION=v1.18.0 GTEST_VERSION=1.8.0 JEMALLOC_VERSION=17c897976c60b0e6e4f4a365c751027244dada7a LZ4_VERSION=v1.7.5 @@ -50,6 +51,7 @@ ZSTD_VERSION=v1.3.7 DEPENDENCIES=( "ARROW_BOOST_URL boost-${BOOST_VERSION}.tar.gz https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION//./_}.tar.gz" "ARROW_BROTLI_URL brotli-${BROTLI_VERSION}.tar.gz https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz" + "ARROW_CARES_URL cares-${CARES_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-$CARES_VERSION.tar.gz" "ARROW_DOUBLE_CONVERSION_URL double-conversion-${DOUBLE_CONVERSION_VERSION}.tar.gz https://github.com/google/double-conversion/archive/${DOUBLE_CONVERSION_VERSION}.tar.gz" "ARROW_FLATBUFFERS_URL flatbuffers-${FLATBUFFERS_VERSION}.tar.gz https://github.com/google/flatbuffers/archive/${FLATBUFFERS_VERSION}.tar.gz" "ARROW_GBENCHMARK_URL gbenchmark-${GBENCHMARK_VERSION}.tar.gz https://github.com/google/benchmark/archive/${GBENCHMARK_VERSION}.tar.gz" From e59bf777bb8babc2fbef4cc881370478acfb8f97 Mon Sep 17 00:00:00 2001 From: Benjamin Kietzman Date: Thu, 31 Jan 2019 11:51:52 -0600 Subject: [PATCH 13/20] ARROW-4430: [C++] Fix untested TypedByteBuffer::Append method Author: Benjamin Kietzman Closes #3528 from bkietz/ARROW-4430-test-typedbufferbuilder-append-copies and squashes the following commits: d70e71de explicitly cast from bool to TypeParam f7397206 fixed format issue 2789a4d4 add test, fix append method --- cpp/src/arrow/buffer-builder.h | 3 ++- cpp/src/arrow/buffer-test.cc | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/buffer-builder.h b/cpp/src/arrow/buffer-builder.h index 9344d5d92b715..b27fbd838f226 100644 --- a/cpp/src/arrow/buffer-builder.h +++ b/cpp/src/arrow/buffer-builder.h @@ -196,7 +196,8 @@ class TypedBufferBuilder::value } Status Append(const int64_t num_copies, T value) { - ARROW_RETURN_NOT_OK(Resize(GrowByFactor(num_copies + length()), false)); + ARROW_RETURN_NOT_OK( + Resize(BufferBuilder::GrowByFactor(num_copies + length()), false)); UnsafeAppend(num_copies, value); return Status::OK(); } diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc index 8ff117402f04c..0154892d12b46 100644 --- a/cpp/src/arrow/buffer-test.cc +++ b/cpp/src/arrow/buffer-test.cc @@ -310,6 +310,22 @@ TYPED_TEST(TypedTestBufferBuilder, BasicTypedBufferBuilderUsage) { } } +TYPED_TEST(TypedTestBufferBuilder, AppendCopies) { + TypedBufferBuilder builder; + + ASSERT_OK(builder.Append(13, static_cast(1))); + ASSERT_OK(builder.Append(17, static_cast(0))); + ASSERT_EQ(builder.length(), 13 + 17); + + std::shared_ptr built; + ASSERT_OK(builder.Finish(&built)); + + auto data = reinterpret_cast(built->data()); + for (int i = 0; i != 13 + 17; ++i, ++data) { + ASSERT_EQ(*data, static_cast(i < 13)) << "index = " << i; + } +} + TEST(TestBufferBuilder, BasicBoolBufferBuilderUsage) { TypedBufferBuilder builder; From 7d1a9e738d54f5fe5659c39b22c77e01f884f747 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Thu, 31 Jan 2019 11:52:30 -0600 Subject: [PATCH 14/20] ARROW-4410: [C++] Fix edge cases in InvertKernel (this also includes documentation fixes which should fix the build due to doxygen warnings) but they are duplciated from ARROW-4411 https://github.com/apache/arrow/pull/3518 Author: Micah Kornfield Closes #3519 from emkornfield/fix_bin_edge_cases_for_real and squashes the following commits: 97e60d47 Fix edge-cases in Binary Kernel Invert e77f67ff remove trailing whitespace 39d93f82 Fix documentation on util-internal as well --- cpp/src/arrow/compute/kernels/boolean-test.cc | 12 ++++++++++++ cpp/src/arrow/compute/kernels/boolean.cc | 12 +++++++----- cpp/src/arrow/compute/kernels/util-internal.cc | 4 ++-- cpp/src/arrow/compute/kernels/util-internal.h | 3 +++ 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/boolean-test.cc b/cpp/src/arrow/compute/kernels/boolean-test.cc index 24b3c68aa1cfb..5f4613367f6c5 100644 --- a/cpp/src/arrow/compute/kernels/boolean-test.cc +++ b/cpp/src/arrow/compute/kernels/boolean-test.cc @@ -130,6 +130,18 @@ TEST_F(TestBooleanKernel, Invert) { ASSERT_TRUE(result_ca->Equals(ca2)); } +TEST_F(TestBooleanKernel, InvertEmptyArray) { + auto type = boolean(); + std::vector> data_buffers(2); + Datum input; + input.value = ArrayData::Make(boolean(), 0 /* length */, std::move(data_buffers), + 0 /* null_count */); + + Datum result; + ASSERT_OK(Invert(&this->ctx_, input, &result)); + ASSERT_TRUE(result.make_array()->Equals(input.make_array())); +} + TEST_F(TestBooleanKernel, And) { vector values1 = {true, false, true, false, true, true}; vector values2 = {true, true, false, false, true, false}; diff --git a/cpp/src/arrow/compute/kernels/boolean.cc b/cpp/src/arrow/compute/kernels/boolean.cc index 91a0e9344305c..78ae7d49bd24f 100644 --- a/cpp/src/arrow/compute/kernels/boolean.cc +++ b/cpp/src/arrow/compute/kernels/boolean.cc @@ -52,7 +52,7 @@ class InvertKernel : public UnaryKernel { // Handle validity bitmap result->null_count = in_data.null_count; const std::shared_ptr& validity_bitmap = in_data.buffers[0]; - if (in_data.offset != 0) { + if (in_data.offset != 0 && in_data.null_count > 0) { DCHECK_LE(BitUtil::BytesForBits(in_data.length), validity_bitmap->size()); CopyBitmap(validity_bitmap->data(), in_data.offset, in_data.length, result->buffers[0]->mutable_data(), kZeroDestOffset); @@ -61,10 +61,12 @@ class InvertKernel : public UnaryKernel { } // Handle output data buffer - const std::shared_ptr& data_buffer = in_data.buffers[1]; - DCHECK_LE(BitUtil::BytesForBits(in_data.length), data_buffer->size()); - InvertBitmap(data_buffer->data(), in_data.offset, in_data.length, - result->buffers[1]->mutable_data(), kZeroDestOffset); + if (in_data.length > 0) { + const Buffer& data_buffer = *in_data.buffers[1]; + DCHECK_LE(BitUtil::BytesForBits(in_data.length), data_buffer.size()); + InvertBitmap(data_buffer.data(), in_data.offset, in_data.length, + result->buffers[1]->mutable_data(), kZeroDestOffset); + } return Status::OK(); } }; diff --git a/cpp/src/arrow/compute/kernels/util-internal.cc b/cpp/src/arrow/compute/kernels/util-internal.cc index 04ee9c02f4957..745b30c3d26a9 100644 --- a/cpp/src/arrow/compute/kernels/util-internal.cc +++ b/cpp/src/arrow/compute/kernels/util-internal.cc @@ -179,8 +179,8 @@ Status PrimitiveAllocatingUnaryKernel::Call(FunctionContext* ctx, const Datum& i MemoryPool* pool = ctx->memory_pool(); // Handle the validity buffer. - if (in_data.offset == 0) { - // Validity bitmap will be zero copied + if (in_data.offset == 0 || in_data.null_count <= 0) { + // Validity bitmap will be zero copied (or allocated when buffer is known). data_buffers.emplace_back(); } else { std::shared_ptr buffer; diff --git a/cpp/src/arrow/compute/kernels/util-internal.h b/cpp/src/arrow/compute/kernels/util-internal.h index 2dd8c0288a7e2..22520235a524c 100644 --- a/cpp/src/arrow/compute/kernels/util-internal.h +++ b/cpp/src/arrow/compute/kernels/util-internal.h @@ -46,6 +46,9 @@ namespace detail { /// \brief Invoke the kernel on value using the ctx and store results in outputs. /// +/// \param[in,out] ctx The function context to use when invoking the kernel. +/// \param[in,out] kernel The kernel to execute. +/// \param[in] value The input value to execute the kernel with. /// \param[out] outputs One ArrayData datum for each ArrayData available in value. ARROW_EXPORT Status InvokeUnaryArrayKernel(FunctionContext* ctx, UnaryKernel* kernel, From 48de821ea79bdf4d0480a2f6b377300ddc5bbd9a Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 31 Jan 2019 23:04:59 -0600 Subject: [PATCH 15/20] ARROW-3846: [Gandiva][C++] Build Gandiva C++ libraries and get unit tests passing on Windows The tests pass cleanly for me on Windows with these changes. Can't say this was the most enjoyable project. I will fix up the CI but wanted to get eyes on the changes in case anything looks undesirable Some notes * This requires that LLVM was built with CMake, which is not true of the Windows installers from llvm.org. It works for me if I `conda install llvmdev=6.0.1 clangdev=6.0.1` * I had to suppress a ton of compiler warnings that seem to be coming from the LLVM headers. see gandiva/llvm_includes.h * Unix targets are still using strptime for date parsing. We might want to use `arrow/vendored/date.h` on all platforms unless there is some significant performance difference * This doesn't build in Appveyor yet. I wanted to wait to see which builds folks think we should add this too (build times will go up because installing llvmdev/clangdev will be time consuming) * Python or other bindings will have to be investigated in a separate patch Author: Wes McKinney Closes #3295 from wesm/gandiva-windows and squashes the following commits: 1fb56714 Add missing file 1e0fdbc8 Revert Flatbuffers changes that are causing CI flakiness 5fb78bc4 Bump flatbuffers version to v1.10.0 707e9d9f Visibility fixes for windows with the cast_time changes fac669e0 Rebase, remove failing unit test f3b60bf3 Some basic fixes to build system, clang IR generation, suppress endogenous LLVM warnings with MSVC 2017 --- appveyor.yml | 1 + ci/appveyor-cpp-build.bat | 7 +++ ci/cpp-msvc-build-main.bat | 1 + cpp/CMakeLists.txt | 6 +- cpp/cmake_modules/FindProtobuf.cmake | 9 ++- cpp/cmake_modules/FindRE2.cmake | 8 ++- cpp/cmake_modules/ThirdpartyToolchain.cmake | 24 ++++---- cpp/src/arrow/util/bit-util-test.cc | 24 ++++++++ cpp/src/arrow/util/bit-util.h | 51 ++++++++++++++++ cpp/src/arrow/util/parsing.h | 2 +- cpp/src/arrow/vendored/datetime.h | 21 +++++++ cpp/src/arrow/vendored/datetime/ios.h | 4 +- cpp/src/arrow/vendored/datetime/tz.cpp | 3 + cpp/src/arrow/vendored/datetime/visibility.h | 26 ++++++++ cpp/src/gandiva/CMakeLists.txt | 23 ++++++- cpp/src/gandiva/annotator.h | 3 +- cpp/src/gandiva/bitmap_accumulator.h | 3 +- cpp/src/gandiva/bitmap_accumulator_test.cc | 16 +++-- cpp/src/gandiva/cast_time.cc | 5 +- cpp/src/gandiva/compiled_expr.h | 2 +- cpp/src/gandiva/configuration.h | 11 ++-- cpp/src/gandiva/date_utils.cc | 3 +- cpp/src/gandiva/date_utils.h | 61 ++++++++++++++++++- cpp/src/gandiva/decimal_type_util.cc | 5 -- cpp/src/gandiva/decimal_type_util.h | 26 ++++---- cpp/src/gandiva/dex.h | 35 +++++------ cpp/src/gandiva/dex_visitor.h | 5 +- cpp/src/gandiva/engine.cc | 14 +++++ cpp/src/gandiva/engine.h | 9 +-- cpp/src/gandiva/expr_decomposer.cc | 6 +- cpp/src/gandiva/expr_decomposer.h | 8 ++- cpp/src/gandiva/expression.h | 3 +- cpp/src/gandiva/expression_registry.h | 5 +- cpp/src/gandiva/filter.cc | 2 + cpp/src/gandiva/filter.h | 12 ++-- cpp/src/gandiva/func_descriptor.h | 3 +- cpp/src/gandiva/function_holder.h | 4 +- cpp/src/gandiva/function_registry.h | 3 +- cpp/src/gandiva/function_signature.h | 3 +- cpp/src/gandiva/like_holder.h | 5 +- cpp/src/gandiva/llvm_generator.h | 3 +- cpp/src/gandiva/llvm_includes.h | 37 +++++++++++ cpp/src/gandiva/llvm_types.h | 6 +- cpp/src/gandiva/lvalue.h | 7 ++- cpp/src/gandiva/native_function.h | 3 +- cpp/src/gandiva/node.h | 13 ++-- cpp/src/gandiva/node_visitor.h | 3 +- cpp/src/gandiva/precompiled/CMakeLists.txt | 33 ++++++++-- .../precompiled/epoch_time_point_test.cc | 37 +++++------ .../gandiva/precompiled/extended_math_ops.cc | 38 +++++++----- cpp/src/gandiva/precompiled/testing.h | 37 +++++++++++ cpp/src/gandiva/precompiled/time_test.cc | 7 +-- cpp/src/gandiva/projector.cc | 2 + cpp/src/gandiva/projector.h | 12 ++-- cpp/src/gandiva/regex_util.h | 3 +- cpp/src/gandiva/selection_vector.cc | 14 ++++- cpp/src/gandiva/selection_vector.h | 3 +- cpp/src/gandiva/tests/date_time_test.cc | 2 +- cpp/src/gandiva/tests/projector_test.cc | 10 ++- cpp/src/gandiva/to_date_holder.cc | 21 ++----- cpp/src/gandiva/to_date_holder.h | 3 +- cpp/src/gandiva/to_date_holder_test.cc | 57 ++++++++++------- cpp/src/gandiva/tree_expr_builder.h | 3 +- cpp/src/gandiva/value_validity_pair.h | 3 +- cpp/src/gandiva/visibility.h | 48 +++++++++++++++ cpp/thirdparty/versions.txt | 2 +- 66 files changed, 650 insertions(+), 219 deletions(-) create mode 100644 cpp/src/arrow/vendored/datetime.h create mode 100644 cpp/src/arrow/vendored/datetime/visibility.h create mode 100644 cpp/src/gandiva/llvm_includes.h create mode 100644 cpp/src/gandiva/precompiled/testing.h create mode 100644 cpp/src/gandiva/visibility.h diff --git a/appveyor.yml b/appveyor.yml index dbf13ff278dc7..d955484ec8362 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -47,6 +47,7 @@ matrix: environment: global: USE_CLCACHE: true + ARROW_BUILD_GANDIVA: "OFF" PYTHON: "3.6" ARCH: "64" diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 78f5e419289cf..f95b88e7bb892 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -104,6 +104,13 @@ conda create -n arrow -q -y -c conda-forge ^ call activate arrow +set ARROW_LLVM_VERSION=6.0.1 + +if "%ARROW_BUILD_GANDIVA%" == "ON" ( + @rem Install llvmdev in the toolchain if building gandiva.dll + conda install -q -y llvmdev=%ARROW_LLVM_VERSION% || exit /B +) + @rem Use Boost from Anaconda set BOOST_ROOT=%CONDA_PREFIX%\Library set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat index ccd64e3d33334..779af154bedb0 100644 --- a/ci/cpp-msvc-build-main.bat +++ b/ci/cpp-msvc-build-main.bat @@ -55,6 +55,7 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ -DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^ -DCMAKE_CXX_FLAGS_RELEASE="/MD %CMAKE_CXX_FLAGS_RELEASE%" ^ + -DARROW_GANDIVA=%ARROW_BUILD_GANDIVA% ^ -DARROW_PARQUET=ON ^ -DARROW_PYTHON=ON ^ .. || exit /B diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3ec430e81a40b..9cdbe7dc801fb 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -309,8 +309,12 @@ Note that this requires linking Boost statically" set(BROTLI_MSVC_STATIC_LIB_SUFFIX "-static" CACHE STRING "Brotli static lib suffix used on Windows with MSVC (default -static)") + set(PROTOBUF_MSVC_STATIC_LIB_SUFFIX "" CACHE STRING + "Protobuf static lib suffix used on Windows with MSVC (default is empty string)") + set(RE2_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING + "re2 static lib suffix used on Windows with MSVC (default is _static)") set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING - "Snappy static lib suffix used on Windows with MSVC (default is empty string)") + "Snappy static lib suffix used on Windows with MSVC (default is _static)") set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING "Lz4 static lib suffix used on Windows with MSVC (default _static)") set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING diff --git a/cpp/cmake_modules/FindProtobuf.cmake b/cpp/cmake_modules/FindProtobuf.cmake index e4a87f4f9cabe..f53f48d60686e 100644 --- a/cpp/cmake_modules/FindProtobuf.cmake +++ b/cpp/cmake_modules/FindProtobuf.cmake @@ -44,12 +44,12 @@ if (EXISTS "${_protobuf_path}/lib/${CMAKE_LIBRARY_ARCHITECTURE}") set (lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}" ${lib_dirs}) endif () -find_library (PROTOBUF_LIBRARY NAMES protobuf PATHS +find_library (PROTOBUF_LIBRARY NAMES protobuf libprotobuf PATHS ${_protobuf_path} NO_DEFAULT_PATH PATH_SUFFIXES ${lib_dirs}) -find_library (PROTOC_LIBRARY NAMES protoc PATHS +find_library (PROTOC_LIBRARY NAMES protoc libprotoc PATHS ${_protobuf_path} NO_DEFAULT_PATH PATH_SUFFIXES ${lib_dirs}) @@ -66,7 +66,7 @@ if (PROTOBUF_INCLUDE_DIR AND PROTOBUF_LIBRARY AND PROTOC_LIBRARY AND PROTOBUF_EX get_filename_component (PROTOBUF_LIBS ${PROTOBUF_LIBRARY} PATH) set (PROTOBUF_LIB_NAME protobuf) set (PROTOC_LIB_NAME protoc) - set (PROTOBUF_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOBUF_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) + set (PROTOBUF_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOBUF_LIB_NAME}${PROTOBUF_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) set (PROTOC_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) else () set (PROTOBUF_FOUND FALSE) @@ -81,7 +81,7 @@ if (PROTOBUF_FOUND) message (STATUS "Found the Protoc executable: ${PROTOBUF_EXECUTABLE}") else() if (_protobuf_path) - set (PROTOBUF_ERR_MSG "Could not find Protobuf. Looked in ${_protobuf_path}.") + set (PROTOBUF_ERR_MSG "Could not find Protobuf. Looked in ${_protobuf_path}") else () set (PROTOBUF_ERR_MSG "Could not find Protobuf in system search paths.") endif() @@ -100,4 +100,3 @@ mark_as_advanced ( PROTOBUF_STATIC_LIB PROTOC_STATIC_LIB ) - diff --git a/cpp/cmake_modules/FindRE2.cmake b/cpp/cmake_modules/FindRE2.cmake index ae0f182d0e48c..51b093fc97767 100644 --- a/cpp/cmake_modules/FindRE2.cmake +++ b/cpp/cmake_modules/FindRE2.cmake @@ -45,14 +45,18 @@ if (EXISTS "${_re2_path}/lib/${CMAKE_LIBRARY_ARCHITECTURE}") set (lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}" ${lib_dirs}) endif () -find_library(RE2_STATIC_LIB NAMES libre2${CMAKE_STATIC_LIBRARY_SUFFIX} +set(RE2_LIB_NAME re2) +set(RE2_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}${RE2_LIB_NAME}${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) +set(RE2_SHARED_LIB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${RE2_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) + +find_library(RE2_STATIC_LIB NAMES ${RE2_STATIC_LIB_NAME} PATHS ${_re2_path} NO_DEFAULT_PATH PATH_SUFFIXES ${lib_dirs} DOC "Google's re2 regex static library" ) -find_library(RE2_SHARED_LIB NAMES libre2${CMAKE_SHARED_LIBRARY_SUFFIX} +find_library(RE2_SHARED_LIB NAMES ${RE2_SHARED_LIB_NAME} PATHS ${_re2_path} NO_DEFAULT_PATH PATH_SUFFIXES ${lib_dirs} diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index ce5073f6d076d..fedeed489b256 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -769,24 +769,22 @@ if (ARROW_WITH_RAPIDJSON) ## Flatbuffers if("${FLATBUFFERS_HOME}" STREQUAL "") - set(FLATBUFFERS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) set(FLATBUFFERS_PREFIX "${THIRDPARTY_PREFIX}") - if (MSVC) - set(FLATBUFFERS_CMAKE_CXX_FLAGS "/EHsc") + set(FLATBUFFERS_CMAKE_CXX_FLAGS /EHsc) + else() + set(FLATBUFFERS_CMAKE_CXX_FLAGS -fPIC) endif() - - # RELEASE build is required for `flatc` to be installed. - set(FLATBUFFERS_BUILD_TYPE RELEASE) - set(FLATBUFFERS_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_BUILD_TYPE=${FLATBUFFERS_BUILD_TYPE} - -DCMAKE_INSTALL_PREFIX=${FLATBUFFERS_PREFIX} - -DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS} - -DFLATBUFFERS_BUILD_TESTS=OFF) - + # We always need to do release builds, otherwise flatc will not be installed. ExternalProject_Add(flatbuffers_ep URL ${FLATBUFFERS_SOURCE_URL} - CMAKE_ARGS ${FLATBUFFERS_CMAKE_ARGS} + CMAKE_ARGS + "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}" + "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}" + "-DFLATBUFFERS_BUILD_TESTS=OFF" + "-DCMAKE_BUILD_TYPE=RELEASE" + "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}" + "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}" ${EP_LOG_OPTIONS}) set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include") diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc index 6709ae4a7d853..6bcb6ea59266b 100644 --- a/cpp/src/arrow/util/bit-util-test.cc +++ b/cpp/src/arrow/util/bit-util-test.cc @@ -788,6 +788,30 @@ TEST(BitUtil, CountLeadingZeros) { EXPECT_EQ(BitUtil::CountLeadingZeros(U64(ULLONG_MAX)), 0); } +TEST(BitUtil, CountTrailingZeros) { + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(0)), 32); + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1) << 31), 31); + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1) << 30), 30); + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1) << 29), 29); + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1) << 28), 28); + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(8)), 3); + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(4)), 2); + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(2)), 1); + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1)), 0); + EXPECT_EQ(BitUtil::CountTrailingZeros(U32(ULONG_MAX)), 0); + + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(0)), 64); + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1) << 63), 63); + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1) << 62), 62); + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1) << 61), 61); + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1) << 60), 60); + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(8)), 3); + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(4)), 2); + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(2)), 1); + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1)), 0); + EXPECT_EQ(BitUtil::CountTrailingZeros(U64(ULLONG_MAX)), 0); +} + #undef U32 #undef U64 diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h index 8e6979ff24b63..bfdb44f255c53 100644 --- a/cpp/src/arrow/util/bit-util.h +++ b/cpp/src/arrow/util/bit-util.h @@ -45,6 +45,7 @@ #if defined(_MSC_VER) #include #pragma intrinsic(_BitScanReverse) +#pragma intrinsic(_BitScanForward) #define ARROW_BYTE_SWAP64 _byteswap_uint64 #define ARROW_BYTE_SWAP32 _byteswap_ulong #else @@ -182,6 +183,56 @@ static inline int CountLeadingZeros(uint64_t value) { #endif } +static inline int CountTrailingZeros(uint32_t value) { +#if defined(__clang__) || defined(__GNUC__) + if (value == 0) return 32; + return static_cast(__builtin_ctzl(value)); +#elif defined(_MSC_VER) + unsigned long index; // NOLINT + if (_BitScanForward(&index, value)) { + return static_cast(index); + } else { + return 32; + } +#else + int bitpos = 0; + if (value) { + while (value & 1 == 0) { + value >>= 1; + ++bitpos; + } + } else { + bitpos = 32; + } + return bitpos; +#endif +} + +static inline int CountTrailingZeros(uint64_t value) { +#if defined(__clang__) || defined(__GNUC__) + if (value == 0) return 64; + return static_cast(__builtin_ctzll(value)); +#elif defined(_MSC_VER) + unsigned long index; // NOLINT + if (_BitScanForward64(&index, value)) { + return static_cast(index); + } else { + return 64; + } +#else + int bitpos = 0; + if (value) { + while (value & 1 == 0) { + value >>= 1; + ++bitpos; + } + } else { + bitpos = 64; + } + return bitpos; +#endif +} + // Returns the minimum number of bits needed to represent an unsigned value static inline int NumRequiredBits(uint64_t x) { return 64 - CountLeadingZeros(x); } diff --git a/cpp/src/arrow/util/parsing.h b/cpp/src/arrow/util/parsing.h index 0d8eb97f873f6..fc6ca0404785c 100644 --- a/cpp/src/arrow/util/parsing.h +++ b/cpp/src/arrow/util/parsing.h @@ -34,7 +34,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/checked_cast.h" -#include "arrow/vendored/datetime/date.h" +#include "arrow/vendored/datetime.h" namespace arrow { namespace internal { diff --git a/cpp/src/arrow/vendored/datetime.h b/cpp/src/arrow/vendored/datetime.h new file mode 100644 index 0000000000000..424313a5f5d14 --- /dev/null +++ b/cpp/src/arrow/vendored/datetime.h @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/vendored/datetime/date.h" +#include "arrow/vendored/datetime/tz.h" diff --git a/cpp/src/arrow/vendored/datetime/ios.h b/cpp/src/arrow/vendored/datetime/ios.h index ec8342f55c9ef..23dc1671aa9fb 100644 --- a/cpp/src/arrow/vendored/datetime/ios.h +++ b/cpp/src/arrow/vendored/datetime/ios.h @@ -40,10 +40,10 @@ { namespace iOSUtils { - + std::string get_tzdata_path(); std::string get_current_timezone(); - + } // namespace iOSUtils } // namespace date } // namespace util diff --git a/cpp/src/arrow/vendored/datetime/tz.cpp b/cpp/src/arrow/vendored/datetime/tz.cpp index ffea8d6173d51..e05423e13c61c 100644 --- a/cpp/src/arrow/vendored/datetime/tz.cpp +++ b/cpp/src/arrow/vendored/datetime/tz.cpp @@ -30,6 +30,9 @@ // been invented (that would involve another several millennia of evolution). // We did not mean to shout. +// wesm: This is required so that symbols are properly exported from the DLL +#include "visibility.h" + #ifdef _WIN32 // windows.h will be included directly and indirectly (e.g. by curl). // We need to define these macros to prevent windows.h bringing in diff --git a/cpp/src/arrow/vendored/datetime/visibility.h b/cpp/src/arrow/vendored/datetime/visibility.h new file mode 100644 index 0000000000000..ae031238d85ac --- /dev/null +++ b/cpp/src/arrow/vendored/datetime/visibility.h @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(ARROW_STATIC) +// intentially empty +#elif defined(ARROW_EXPORTING) +#define DATE_BUILD_DLL +#else +#define DATE_USE_DLL +#endif diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 52784e76ace69..d5f4364129b18 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -32,8 +32,8 @@ set(GANDIVA_BC_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/gandiva) set(GANDIVA_BC_FILE_NAME irhelpers.bc) -set(GANDIVA_BC_INSTALL_PATH ${GANDIVA_BC_INSTALL_DIR}/${GANDIVA_BC_FILE_NAME}) -set(GANDIVA_BC_OUTPUT_PATH ${CMAKE_CURRENT_BINARY_DIR}/${GANDIVA_BC_FILE_NAME}) +set(GANDIVA_BC_INSTALL_PATH "${GANDIVA_BC_INSTALL_DIR}/${GANDIVA_BC_FILE_NAME}") +set(GANDIVA_BC_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${GANDIVA_BC_FILE_NAME}") install(FILES ${GANDIVA_BC_OUTPUT_PATH} DESTINATION ${GANDIVA_BC_INSTALL_DIR}) @@ -95,16 +95,35 @@ if (ARROW_GANDIVA_STATIC_LIBSTDCPP -static-libgcc) endif() +# if (MSVC) +# # Symbols that need to be made public in gandiva.dll for LLVM IR +# # compilation +# set(MSVC_SYMBOL_EXPORTS _Init_thread_header) +# foreach(SYMBOL ${MSVC_SYMBOL_EXPORTS}) +# set(GANDIVA_SHARED_LINK_FLAGS "${GANDIVA_SHARED_LINK_FLAGS} /EXPORT:${SYMBOL}") +# endforeach() +# endif() + ADD_ARROW_LIB(gandiva SOURCES ${SRC_FILES} OUTPUTS GANDIVA_LIBRARIES DEPENDENCIES arrow_dependencies precompiled EXTRA_INCLUDES $ + SHARED_LINK_FLAGS ${GANDIVA_SHARED_LINK_FLAGS} SHARED_LINK_LIBS arrow_shared SHARED_PRIVATE_LINK_LIBS ${GANDIVA_SHARED_PRIVATE_LINK_LIBS} STATIC_LINK_LIBS ${GANDIVA_STATIC_LINK_LIBS}) +foreach(LIB_TARGET ${GANDIVA_LIBRARIES}) + target_compile_definitions(${LIB_TARGET} + PRIVATE GANDIVA_EXPORTING) +endforeach() + +if (ARROW_BUILD_STATIC AND WIN32) + target_compile_definitions(gandiva_static PUBLIC GANDIVA_STATIC) +endif() + add_dependencies(gandiva ${GANDIVA_LIBRARIES}) # install for gandiva diff --git a/cpp/src/gandiva/annotator.h b/cpp/src/gandiva/annotator.h index 6c2cd05b04efd..c0ddc02463590 100644 --- a/cpp/src/gandiva/annotator.h +++ b/cpp/src/gandiva/annotator.h @@ -27,12 +27,13 @@ #include "gandiva/eval_batch.h" #include "gandiva/gandiva_aliases.h" #include "gandiva/logging.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief annotate the arrow fields in an expression, and use that /// to convert the incoming arrow-format row batch to an EvalBatch. -class Annotator { +class GANDIVA_EXPORT Annotator { public: Annotator() : buffer_count_(0), local_bitmap_count_(0) {} diff --git a/cpp/src/gandiva/bitmap_accumulator.h b/cpp/src/gandiva/bitmap_accumulator.h index 157405d680e5b..15a2044b5fd4d 100644 --- a/cpp/src/gandiva/bitmap_accumulator.h +++ b/cpp/src/gandiva/bitmap_accumulator.h @@ -24,12 +24,13 @@ #include "gandiva/dex.h" #include "gandiva/dex_visitor.h" #include "gandiva/eval_batch.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief Extract bitmap buffer from either the input/buffer vectors or the /// local validity bitmap, and accumultes them to do the final computation. -class BitMapAccumulator : public DexDefaultVisitor { +class GANDIVA_EXPORT BitMapAccumulator : public DexDefaultVisitor { public: explicit BitMapAccumulator(const EvalBatch& eval_batch) : eval_batch_(eval_batch), all_invalid_(false) {} diff --git a/cpp/src/gandiva/bitmap_accumulator_test.cc b/cpp/src/gandiva/bitmap_accumulator_test.cc index 53e8aaca21ff1..51a8b09ec724d 100644 --- a/cpp/src/gandiva/bitmap_accumulator_test.cc +++ b/cpp/src/gandiva/bitmap_accumulator_test.cc @@ -21,24 +21,22 @@ #include #include + +#include "arrow/test-util.h" + #include "gandiva/dex.h" namespace gandiva { class TestBitMapAccumulator : public ::testing::Test { protected: - void FillBitMap(uint8_t* bmap, int nrecords); + void FillBitMap(uint8_t* bmap, uint32_t seed, int nrecords); void ByteWiseIntersectBitMaps(uint8_t* dst, const std::vector& srcs, int nrecords); }; -void TestBitMapAccumulator::FillBitMap(uint8_t* bmap, int nbytes) { - unsigned int cur = 0; - - for (int i = 0; i < nbytes; ++i) { - rand_r(&cur); - bmap[i] = static_cast(cur % UINT8_MAX); - } +void TestBitMapAccumulator::FillBitMap(uint8_t* bmap, uint32_t seed, int nbytes) { + ::arrow::random_bytes(nbytes, seed, bmap); } void TestBitMapAccumulator::ByteWiseIntersectBitMaps(uint8_t* dst, @@ -61,7 +59,7 @@ TEST_F(TestBitMapAccumulator, TestIntersectBitMaps) { uint8_t expected_bitmap[length]; for (int i = 0; i < 4; i++) { - FillBitMap(src_bitmaps[i], length); + FillBitMap(src_bitmaps[i], i, length); } for (int i = 0; i < 4; i++) { diff --git a/cpp/src/gandiva/cast_time.cc b/cpp/src/gandiva/cast_time.cc index ee3fd316eeeee..1d4293b199661 100644 --- a/cpp/src/gandiva/cast_time.cc +++ b/cpp/src/gandiva/cast_time.cc @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/vendored/datetime/tz.h" +#include + +#include "arrow/vendored/datetime.h" + #include "gandiva/precompiled/time_fields.h" #ifndef GANDIVA_UNIT_TEST diff --git a/cpp/src/gandiva/compiled_expr.h b/cpp/src/gandiva/compiled_expr.h index 2f23971f366d3..b7799f18928e0 100644 --- a/cpp/src/gandiva/compiled_expr.h +++ b/cpp/src/gandiva/compiled_expr.h @@ -18,7 +18,7 @@ #ifndef GANDIVA_COMPILED_EXPR_H #define GANDIVA_COMPILED_EXPR_H -#include +#include "gandiva/llvm_includes.h" #include "gandiva/value_validity_pair.h" namespace gandiva { diff --git a/cpp/src/gandiva/configuration.h b/cpp/src/gandiva/configuration.h index 04e2eed287e13..480a95e9274ee 100644 --- a/cpp/src/gandiva/configuration.h +++ b/cpp/src/gandiva/configuration.h @@ -15,16 +15,18 @@ // specific language governing permissions and limitations // under the License. -#ifndef GANDIVA_CONFIGURATION_H -#define GANDIVA_CONFIGURATION_H +#pragma once #include #include #include "arrow/status.h" +#include "gandiva/visibility.h" + namespace gandiva { +GANDIVA_EXPORT extern const char kByteCodeFilePath[]; class ConfigurationBuilder; @@ -32,7 +34,7 @@ class ConfigurationBuilder; /// /// It contains elements to customize gandiva execution /// at run time. -class Configuration { +class GANDIVA_EXPORT Configuration { public: friend class ConfigurationBuilder; @@ -53,7 +55,7 @@ class Configuration { /// /// Provides a default configuration and convenience methods /// to override specific values and build a custom instance -class ConfigurationBuilder { +class GANDIVA_EXPORT ConfigurationBuilder { public: ConfigurationBuilder() : byte_code_file_path_(kByteCodeFilePath) {} @@ -83,4 +85,3 @@ class ConfigurationBuilder { }; } // namespace gandiva -#endif // GANDIVA_CONFIGURATION_H diff --git a/cpp/src/gandiva/date_utils.cc b/cpp/src/gandiva/date_utils.cc index 8a7e1f03fbd20..f0a80d3c95921 100644 --- a/cpp/src/gandiva/date_utils.cc +++ b/cpp/src/gandiva/date_utils.cc @@ -16,6 +16,7 @@ // under the License. #include +#include #include #include #include @@ -57,7 +58,7 @@ Status DateUtils::ToInternalFormat(const std::string& format, std::stringstream buffer; bool is_in_quoted_text = false; - for (uint i = 0; i < format.length(); i++) { + for (size_t i = 0; i < format.size(); i++) { char currentChar = format[i]; // logic before we append to the buffer diff --git a/cpp/src/gandiva/date_utils.h b/cpp/src/gandiva/date_utils.h index 64a150b6ba72d..e87203bd017ee 100644 --- a/cpp/src/gandiva/date_utils.h +++ b/cpp/src/gandiva/date_utils.h @@ -23,12 +23,22 @@ #include #include +#if defined(_MSC_VER) +#include +#include +#include +#endif + +#include "arrow/util/macros.h" +#include "arrow/vendored/datetime.h" + #include "gandiva/arrow.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief Utility class for converting sql date patterns to internal date patterns. -class DateUtils { +class GANDIVA_EXPORT DateUtils { public: static Status ToInternalFormat(const std::string& format, std::shared_ptr* internal_format); @@ -47,6 +57,55 @@ class DateUtils { static std::vector GetExactMatches(const std::string& pattern); }; +namespace internal { + +/// \brief Returns seconds since the UNIX epoch +static inline bool ParseTimestamp(const char* buf, const char* format, + bool ignoreTimeInDay, int64_t* out) { +#if defined(_MSC_VER) + static std::locale lc_all(setlocale(LC_ALL, NULLPTR)); + std::istringstream stream(buf); + stream.imbue(lc_all); + + // TODO: date::parse fails parsing when the hour value is 0. + // eg.1886-12-01 00:00:00 + arrow::util::date::sys_seconds seconds; + if (ignoreTimeInDay) { + arrow::util::date::sys_days days; + stream >> arrow::util::date::parse(format, days); + if (stream.fail()) { + return false; + } + seconds = days; + } else { + stream >> arrow::util::date::parse(format, seconds); + if (stream.fail()) { + return false; + } + } + auto seconds_in_epoch = seconds.time_since_epoch().count(); + *out = seconds_in_epoch; + return true; +#else + struct tm result; + char* ret = strptime(buf, format, &result); + if (ret == NULLPTR) { + return false; + } + // ignore the time part + arrow::util::date::sys_seconds secs = + arrow::util::date::sys_days(arrow::util::date::year(result.tm_year + 1900) / + (result.tm_mon + 1) / result.tm_mday); + if (!ignoreTimeInDay) { + secs += (std::chrono::hours(result.tm_hour) + std::chrono::minutes(result.tm_min) + + std::chrono::seconds(result.tm_sec)); + } + *out = secs.time_since_epoch().count(); + return true; +#endif +} + +} // namespace internal } // namespace gandiva #endif // TO_DATE_HELPER_H diff --git a/cpp/src/gandiva/decimal_type_util.cc b/cpp/src/gandiva/decimal_type_util.cc index 2795e913a9484..74c9326176373 100644 --- a/cpp/src/gandiva/decimal_type_util.cc +++ b/cpp/src/gandiva/decimal_type_util.cc @@ -20,11 +20,6 @@ namespace gandiva { -constexpr int32_t DecimalTypeUtil::kMaxDecimal32Precision; -constexpr int32_t DecimalTypeUtil::kMaxDecimal64Precision; -constexpr int32_t DecimalTypeUtil::kMaxPrecision; - -constexpr int32_t DecimalTypeUtil::kMaxScale; constexpr int32_t DecimalTypeUtil::kMinAdjustedScale; #define DCHECK_TYPE(type) \ diff --git a/cpp/src/gandiva/decimal_type_util.h b/cpp/src/gandiva/decimal_type_util.h index 2c095c159bba0..aa3c255bb6948 100644 --- a/cpp/src/gandiva/decimal_type_util.h +++ b/cpp/src/gandiva/decimal_type_util.h @@ -24,12 +24,13 @@ #include #include "gandiva/arrow.h" +#include "gandiva/visibility.h" namespace gandiva { /// @brief Handles conversion of scale/precision for operations on decimal types. /// TODO : do validations for all of these. -class DecimalTypeUtil { +class GANDIVA_EXPORT DecimalTypeUtil { public: enum Op { kOpAdd, @@ -65,7 +66,16 @@ class DecimalTypeUtil { static Decimal128TypePtr MakeType(int32_t precision, int32_t scale); private: - static Decimal128TypePtr MakeAdjustedType(int32_t precision, int32_t scale); + // Reduce the scale if possible so that precision stays <= kMaxPrecision + static Decimal128TypePtr MakeAdjustedType(int32_t precision, int32_t scale) { + if (precision > kMaxPrecision) { + int32_t min_scale = std::min(scale, kMinAdjustedScale); + int32_t delta = precision - kMaxPrecision; + precision = kMaxPrecision; + scale = std::max(scale - delta, min_scale); + } + return MakeType(precision, scale); + } }; inline Decimal128TypePtr DecimalTypeUtil::MakeType(int32_t precision, int32_t scale) { @@ -73,18 +83,6 @@ inline Decimal128TypePtr DecimalTypeUtil::MakeType(int32_t precision, int32_t sc arrow::decimal(precision, scale)); } -// Reduce the scale if possible so that precision stays <= kMaxPrecision -inline Decimal128TypePtr DecimalTypeUtil::MakeAdjustedType(int32_t precision, - int32_t scale) { - if (precision > kMaxPrecision) { - int32_t min_scale = std::min(scale, kMinAdjustedScale); - int32_t delta = precision - kMaxPrecision; - precision = kMaxPrecision; - scale = std::max(scale - delta, min_scale); - } - return MakeType(precision, scale); -} - } // namespace gandiva #endif // GANDIVA_DECIMAL_TYPE_SQL_H diff --git a/cpp/src/gandiva/dex.h b/cpp/src/gandiva/dex.h index afce44ed12fee..894d9611058bd 100644 --- a/cpp/src/gandiva/dex.h +++ b/cpp/src/gandiva/dex.h @@ -32,11 +32,12 @@ #include "gandiva/literal_holder.h" #include "gandiva/native_function.h" #include "gandiva/value_validity_pair.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief Decomposed expression : the validity and value are separated. -class Dex { +class GANDIVA_EXPORT Dex { public: /// Derived classes should simply invoke the Visit api of the visitor. virtual void Accept(DexVisitor& visitor) = 0; @@ -44,7 +45,7 @@ class Dex { }; /// Base class for other Vector related Dex. -class VectorReadBaseDex : public Dex { +class GANDIVA_EXPORT VectorReadBaseDex : public Dex { public: explicit VectorReadBaseDex(FieldDescriptorPtr field_desc) : field_desc_(field_desc) {} @@ -59,7 +60,7 @@ class VectorReadBaseDex : public Dex { }; /// validity component of a ValueVector -class VectorReadValidityDex : public VectorReadBaseDex { +class GANDIVA_EXPORT VectorReadValidityDex : public VectorReadBaseDex { public: explicit VectorReadValidityDex(FieldDescriptorPtr field_desc) : VectorReadBaseDex(field_desc) {} @@ -70,7 +71,7 @@ class VectorReadValidityDex : public VectorReadBaseDex { }; /// value component of a fixed-len ValueVector -class VectorReadFixedLenValueDex : public VectorReadBaseDex { +class GANDIVA_EXPORT VectorReadFixedLenValueDex : public VectorReadBaseDex { public: explicit VectorReadFixedLenValueDex(FieldDescriptorPtr field_desc) : VectorReadBaseDex(field_desc) {} @@ -81,7 +82,7 @@ class VectorReadFixedLenValueDex : public VectorReadBaseDex { }; /// value component of a variable-len ValueVector -class VectorReadVarLenValueDex : public VectorReadBaseDex { +class GANDIVA_EXPORT VectorReadVarLenValueDex : public VectorReadBaseDex { public: explicit VectorReadVarLenValueDex(FieldDescriptorPtr field_desc) : VectorReadBaseDex(field_desc) {} @@ -94,7 +95,7 @@ class VectorReadVarLenValueDex : public VectorReadBaseDex { }; /// validity based on a local bitmap. -class LocalBitMapValidityDex : public Dex { +class GANDIVA_EXPORT LocalBitMapValidityDex : public Dex { public: explicit LocalBitMapValidityDex(int local_bitmap_idx) : local_bitmap_idx_(local_bitmap_idx) {} @@ -108,7 +109,7 @@ class LocalBitMapValidityDex : public Dex { }; /// base function expression -class FuncDex : public Dex { +class GANDIVA_EXPORT FuncDex : public Dex { public: FuncDex(FuncDescriptorPtr func_descriptor, const NativeFunction* native_function, FunctionHolderPtr function_holder, const ValueValidityPairVector& args) @@ -134,7 +135,7 @@ class FuncDex : public Dex { /// A function expression that only deals with non-null inputs, and generates non-null /// outputs. -class NonNullableFuncDex : public FuncDex { +class GANDIVA_EXPORT NonNullableFuncDex : public FuncDex { public: NonNullableFuncDex(FuncDescriptorPtr func_descriptor, const NativeFunction* native_function, @@ -147,7 +148,7 @@ class NonNullableFuncDex : public FuncDex { /// A function expression that deals with nullable inputs, but generates non-null /// outputs. -class NullableNeverFuncDex : public FuncDex { +class GANDIVA_EXPORT NullableNeverFuncDex : public FuncDex { public: NullableNeverFuncDex(FuncDescriptorPtr func_descriptor, const NativeFunction* native_function, @@ -160,7 +161,7 @@ class NullableNeverFuncDex : public FuncDex { /// A function expression that deals with nullable inputs, and /// nullable outputs. -class NullableInternalFuncDex : public FuncDex { +class GANDIVA_EXPORT NullableInternalFuncDex : public FuncDex { public: NullableInternalFuncDex(FuncDescriptorPtr func_descriptor, const NativeFunction* native_function, @@ -179,17 +180,17 @@ class NullableInternalFuncDex : public FuncDex { }; /// special validity type that always returns true. -class TrueDex : public Dex { +class GANDIVA_EXPORT TrueDex : public Dex { void Accept(DexVisitor& visitor) override { visitor.Visit(*this); } }; /// special validity type that always returns false. -class FalseDex : public Dex { +class GANDIVA_EXPORT FalseDex : public Dex { void Accept(DexVisitor& visitor) override { visitor.Visit(*this); } }; /// decomposed expression for a literal. -class LiteralDex : public Dex { +class GANDIVA_EXPORT LiteralDex : public Dex { public: LiteralDex(DataTypePtr type, const LiteralHolder& holder) : type_(type), holder_(holder) {} @@ -206,7 +207,7 @@ class LiteralDex : public Dex { }; /// decomposed if-else expression. -class IfDex : public Dex { +class GANDIVA_EXPORT IfDex : public Dex { public: IfDex(ValueValidityPairPtr condition_vv, ValueValidityPairPtr then_vv, ValueValidityPairPtr else_vv, DataTypePtr result_type, int local_bitmap_idx, @@ -242,7 +243,7 @@ class IfDex : public Dex { }; // decomposed boolean expression. -class BooleanDex : public Dex { +class GANDIVA_EXPORT BooleanDex : public Dex { public: BooleanDex(const ValueValidityPairVector& args, int local_bitmap_idx) : args_(args), local_bitmap_idx_(local_bitmap_idx) {} @@ -258,7 +259,7 @@ class BooleanDex : public Dex { }; /// Boolean-AND expression -class BooleanAndDex : public BooleanDex { +class GANDIVA_EXPORT BooleanAndDex : public BooleanDex { public: BooleanAndDex(const ValueValidityPairVector& args, int local_bitmap_idx) : BooleanDex(args, local_bitmap_idx) {} @@ -267,7 +268,7 @@ class BooleanAndDex : public BooleanDex { }; /// Boolean-OR expression -class BooleanOrDex : public BooleanDex { +class GANDIVA_EXPORT BooleanOrDex : public BooleanDex { public: BooleanOrDex(const ValueValidityPairVector& args, int local_bitmap_idx) : BooleanDex(args, local_bitmap_idx) {} diff --git a/cpp/src/gandiva/dex_visitor.h b/cpp/src/gandiva/dex_visitor.h index 456fe430511dc..c34629a53e1a8 100644 --- a/cpp/src/gandiva/dex_visitor.h +++ b/cpp/src/gandiva/dex_visitor.h @@ -21,6 +21,7 @@ #include #include "gandiva/logging.h" +#include "gandiva/visibility.h" namespace gandiva { @@ -41,7 +42,7 @@ template class InExprDexBase; /// \brief Visitor for decomposed expression. -class DexVisitor { +class GANDIVA_EXPORT DexVisitor { public: virtual ~DexVisitor() = default; @@ -67,7 +68,7 @@ class DexVisitor { #define VISIT_DCHECK(DEX_CLASS) \ void Visit(const DEX_CLASS& dex) override { DCHECK(0); } -class DexDefaultVisitor : public DexVisitor { +class GANDIVA_EXPORT DexDefaultVisitor : public DexVisitor { VISIT_DCHECK(VectorReadValidityDex) VISIT_DCHECK(VectorReadFixedLenValueDex) VISIT_DCHECK(VectorReadVarLenValueDex) diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index 9aaafea8e498e..d073a3e749f8a 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -23,6 +23,15 @@ #include #include +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4141) +#pragma warning(disable : 4146) +#pragma warning(disable : 4244) +#pragma warning(disable : 4267) +#pragma warning(disable : 4624) +#endif + #include #include #include @@ -39,6 +48,11 @@ #include #include #include + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + #include "gandiva/decimal_ir.h" #include "gandiva/exported_funcs_registry.h" diff --git a/cpp/src/gandiva/engine.h b/cpp/src/gandiva/engine.h index 12480148bf422..7a976d5f9265c 100644 --- a/cpp/src/gandiva/engine.h +++ b/cpp/src/gandiva/engine.h @@ -23,24 +23,21 @@ #include #include -#include -#include -#include -#include - #include "arrow/status.h" #include "arrow/util/macros.h" #include "gandiva/configuration.h" +#include "gandiva/llvm_includes.h" #include "gandiva/llvm_types.h" #include "gandiva/logging.h" +#include "gandiva/visibility.h" namespace gandiva { class FunctionIRBuilder; /// \brief LLVM Execution engine wrapper. -class Engine { +class GANDIVA_EXPORT Engine { public: llvm::LLVMContext* context() { return context_.get(); } llvm::IRBuilder<>* ir_builder() { return ir_builder_.get(); } diff --git a/cpp/src/gandiva/expr_decomposer.cc b/cpp/src/gandiva/expr_decomposer.cc index bed84ededb5e7..91014f1b82783 100644 --- a/cpp/src/gandiva/expr_decomposer.cc +++ b/cpp/src/gandiva/expr_decomposer.cc @@ -232,7 +232,7 @@ int ExprDecomposer::PushThenEntry(const IfNode& node) { // push new entry to the stack. std::unique_ptr entry(new IfStackEntry( node, kStackEntryThen, false /*is_terminal_else*/, local_bitmap_idx)); - if_entries_stack_.push(std::move(entry)); + if_entries_stack_.emplace(std::move(entry)); return local_bitmap_idx; } @@ -250,7 +250,7 @@ void ExprDecomposer::PopThenEntry(const IfNode& node) { void ExprDecomposer::PushElseEntry(const IfNode& node, int local_bitmap_idx) { std::unique_ptr entry(new IfStackEntry( node, kStackEntryElse, true /*is_terminal_else*/, local_bitmap_idx)); - if_entries_stack_.push(std::move(entry)); + if_entries_stack_.emplace(std::move(entry)); } bool ExprDecomposer::PopElseEntry(const IfNode& node) { @@ -268,7 +268,7 @@ bool ExprDecomposer::PopElseEntry(const IfNode& node) { void ExprDecomposer::PushConditionEntry(const IfNode& node) { std::unique_ptr entry(new IfStackEntry(node, kStackEntryCondition)); - if_entries_stack_.push(std::move(entry)); + if_entries_stack_.emplace(std::move(entry)); } void ExprDecomposer::PopConditionEntry(const IfNode& node) { diff --git a/cpp/src/gandiva/expr_decomposer.h b/cpp/src/gandiva/expr_decomposer.h index bc21ed07cf57c..ab92ca3d24940 100644 --- a/cpp/src/gandiva/expr_decomposer.h +++ b/cpp/src/gandiva/expr_decomposer.h @@ -27,6 +27,7 @@ #include "gandiva/expression.h" #include "gandiva/node.h" #include "gandiva/node_visitor.h" +#include "gandiva/visibility.h" namespace gandiva { @@ -35,7 +36,7 @@ class Annotator; /// \brief Decomposes an expression tree to seperate out the validity and /// value expressions. -class ExprDecomposer : public NodeVisitor { +class GANDIVA_EXPORT ExprDecomposer : public NodeVisitor { public: explicit ExprDecomposer(const FunctionRegistry& registry, Annotator& annotator) : registry_(registry), annotator_(annotator) {} @@ -49,6 +50,8 @@ class ExprDecomposer : public NodeVisitor { } private: + ARROW_DISALLOW_COPY_AND_ASSIGN(ExprDecomposer); + FRIEND_TEST(TestExprDecomposer, TestStackSimple); FRIEND_TEST(TestExprDecomposer, TestNested); FRIEND_TEST(TestExprDecomposer, TestInternalIf); @@ -83,6 +86,9 @@ class ExprDecomposer : public NodeVisitor { StackEntryType entry_type_; bool is_terminal_else_; int local_bitmap_idx_; + + private: + ARROW_DISALLOW_COPY_AND_ASSIGN(IfStackEntry); }; // pop 'condition entry' into stack. diff --git a/cpp/src/gandiva/expression.h b/cpp/src/gandiva/expression.h index e3ae18f4d4c28..2141e871393e5 100644 --- a/cpp/src/gandiva/expression.h +++ b/cpp/src/gandiva/expression.h @@ -22,11 +22,12 @@ #include "gandiva/arrow.h" #include "gandiva/gandiva_aliases.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief An expression tree with a root node, and a result field. -class Expression { +class GANDIVA_EXPORT Expression { public: Expression(const NodePtr root, const FieldPtr result) : root_(root), result_(result) {} diff --git a/cpp/src/gandiva/expression_registry.h b/cpp/src/gandiva/expression_registry.h index a03deab91cdc1..4524a077a629c 100644 --- a/cpp/src/gandiva/expression_registry.h +++ b/cpp/src/gandiva/expression_registry.h @@ -24,6 +24,7 @@ #include "gandiva/arrow.h" #include "gandiva/function_signature.h" #include "gandiva/gandiva_aliases.h" +#include "gandiva/visibility.h" namespace gandiva { @@ -33,13 +34,13 @@ class FunctionRegistry; /// /// Has helper methods for clients to programatically discover /// data types and functions supported by Gandiva. -class ExpressionRegistry { +class GANDIVA_EXPORT ExpressionRegistry { public: using iterator = const NativeFunction*; ExpressionRegistry(); ~ExpressionRegistry(); static DataTypeVector supported_types() { return supported_types_; } - class FunctionSignatureIterator { + class GANDIVA_EXPORT FunctionSignatureIterator { public: explicit FunctionSignatureIterator(iterator it) : it_(it) {} diff --git a/cpp/src/gandiva/filter.cc b/cpp/src/gandiva/filter.cc index 6075e2574559b..3bba1909af866 100644 --- a/cpp/src/gandiva/filter.cc +++ b/cpp/src/gandiva/filter.cc @@ -37,6 +37,8 @@ Filter::Filter(std::unique_ptr llvm_generator, SchemaPtr schema, schema_(schema), configuration_(configuration) {} +Filter::~Filter() {} + Status Filter::Make(SchemaPtr schema, ConditionPtr condition, std::shared_ptr configuration, std::shared_ptr* filter) { diff --git a/cpp/src/gandiva/filter.h b/cpp/src/gandiva/filter.h index 6ff7010ac07f6..4fbda806e0af9 100644 --- a/cpp/src/gandiva/filter.h +++ b/cpp/src/gandiva/filter.h @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -#ifndef GANDIVA_EXPR_FILTER_H -#define GANDIVA_EXPR_FILTER_H +#pragma once #include #include @@ -29,6 +28,7 @@ #include "gandiva/condition.h" #include "gandiva/configuration.h" #include "gandiva/selection_vector.h" +#include "gandiva/visibility.h" namespace gandiva { @@ -38,12 +38,14 @@ class LLVMGenerator; /// /// A filter is built for a specific schema and condition. Once the filter is built, it /// can be used to evaluate many row batches. -class Filter { +class GANDIVA_EXPORT Filter { public: Filter(std::unique_ptr llvm_generator, SchemaPtr schema, std::shared_ptr config); - ~Filter() = default; + // Inline dtor will attempt to resolve the destructor for + // LLVMGenerator on MSVC, so we compile the dtor in the object code + ~Filter(); /// Build a filter for the given schema and condition, with the default configuration. /// @@ -81,5 +83,3 @@ class Filter { }; } // namespace gandiva - -#endif // GANDIVA_EXPR_FILTER_H diff --git a/cpp/src/gandiva/func_descriptor.h b/cpp/src/gandiva/func_descriptor.h index 9b18a9b694319..08f719995fe6b 100644 --- a/cpp/src/gandiva/func_descriptor.h +++ b/cpp/src/gandiva/func_descriptor.h @@ -22,11 +22,12 @@ #include #include "gandiva/arrow.h" +#include "gandiva/visibility.h" namespace gandiva { /// Descriptor for a function in the expression. -class FuncDescriptor { +class GANDIVA_EXPORT FuncDescriptor { public: FuncDescriptor(const std::string& name, const DataTypeVector& params, DataTypePtr return_type) diff --git a/cpp/src/gandiva/function_holder.h b/cpp/src/gandiva/function_holder.h index 4d007d1db3fa7..43dbeac07c97f 100644 --- a/cpp/src/gandiva/function_holder.h +++ b/cpp/src/gandiva/function_holder.h @@ -20,10 +20,12 @@ #include +#include "gandiva/visibility.h" + namespace gandiva { /// Holder for a function that can be invoked from LLVM. -class FunctionHolder { +class GANDIVA_EXPORT FunctionHolder { public: virtual ~FunctionHolder() = default; }; diff --git a/cpp/src/gandiva/function_registry.h b/cpp/src/gandiva/function_registry.h index 810bf2d3eb338..f7aa3de4bb50a 100644 --- a/cpp/src/gandiva/function_registry.h +++ b/cpp/src/gandiva/function_registry.h @@ -22,11 +22,12 @@ #include "gandiva/function_registry_common.h" #include "gandiva/gandiva_aliases.h" #include "gandiva/native_function.h" +#include "gandiva/visibility.h" namespace gandiva { ///\brief Registry of pre-compiled IR functions. -class FunctionRegistry { +class GANDIVA_EXPORT FunctionRegistry { public: using iterator = const NativeFunction*; diff --git a/cpp/src/gandiva/function_signature.h b/cpp/src/gandiva/function_signature.h index ee82abc367e20..a5015ce43ec75 100644 --- a/cpp/src/gandiva/function_signature.h +++ b/cpp/src/gandiva/function_signature.h @@ -24,12 +24,13 @@ #include "gandiva/arrow.h" #include "gandiva/logging.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief Signature for a function : includes the base name, input param types and /// output types. -class FunctionSignature { +class GANDIVA_EXPORT FunctionSignature { public: FunctionSignature(const std::string& base_name, const DataTypeVector& param_types, DataTypePtr ret_type) diff --git a/cpp/src/gandiva/like_holder.h b/cpp/src/gandiva/like_holder.h index 23ed367e8ccf7..eab30bf732fa4 100644 --- a/cpp/src/gandiva/like_holder.h +++ b/cpp/src/gandiva/like_holder.h @@ -22,14 +22,17 @@ #include #include + #include "arrow/status.h" + #include "gandiva/function_holder.h" #include "gandiva/node.h" +#include "gandiva/visibility.h" namespace gandiva { /// Function Holder for SQL 'like' -class LikeHolder : public FunctionHolder { +class GANDIVA_EXPORT LikeHolder : public FunctionHolder { public: ~LikeHolder() override = default; diff --git a/cpp/src/gandiva/llvm_generator.h b/cpp/src/gandiva/llvm_generator.h index 937e5acc87b2e..2c1d5c10194ac 100644 --- a/cpp/src/gandiva/llvm_generator.h +++ b/cpp/src/gandiva/llvm_generator.h @@ -36,13 +36,14 @@ #include "gandiva/llvm_types.h" #include "gandiva/lvalue.h" #include "gandiva/value_validity_pair.h" +#include "gandiva/visibility.h" namespace gandiva { class FunctionHolder; /// Builds an LLVM module and generates code for the specified set of expressions. -class LLVMGenerator { +class GANDIVA_EXPORT LLVMGenerator { public: /// \brief Factory method to initialize the generator. static Status Make(std::shared_ptr config, diff --git a/cpp/src/gandiva/llvm_includes.h b/cpp/src/gandiva/llvm_includes.h new file mode 100644 index 0000000000000..9de1f45a0ad61 --- /dev/null +++ b/cpp/src/gandiva/llvm_includes.h @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4141) +#pragma warning(disable : 4146) +#pragma warning(disable : 4244) +#pragma warning(disable : 4267) +#pragma warning(disable : 4291) +#pragma warning(disable : 4624) +#endif + +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif diff --git a/cpp/src/gandiva/llvm_types.h b/cpp/src/gandiva/llvm_types.h index 9cf4dd5d1c850..2629d326c3590 100644 --- a/cpp/src/gandiva/llvm_types.h +++ b/cpp/src/gandiva/llvm_types.h @@ -21,15 +21,15 @@ #include #include -#include -#include #include "gandiva/arrow.h" +#include "gandiva/llvm_includes.h" #include "gandiva/logging.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief Holder for llvm types, and mappings between arrow types and llvm types. -class LLVMTypes { +class GANDIVA_EXPORT LLVMTypes { public: explicit LLVMTypes(llvm::LLVMContext& context); diff --git a/cpp/src/gandiva/lvalue.h b/cpp/src/gandiva/lvalue.h index ce5040f6c37a6..6c9814cd63017 100644 --- a/cpp/src/gandiva/lvalue.h +++ b/cpp/src/gandiva/lvalue.h @@ -20,14 +20,15 @@ #include -#include #include "arrow/util/macros.h" + +#include "gandiva/llvm_includes.h" #include "gandiva/logging.h" namespace gandiva { /// \brief Tracks validity/value builders in LLVM. -class LValue { +class GANDIVA_EXPORT LValue { public: explicit LValue(llvm::Value* data, llvm::Value* length = NULLPTR, llvm::Value* validity = NULLPTR) @@ -54,7 +55,7 @@ class LValue { llvm::Value* validity_; }; -class DecimalLValue : public LValue { +class GANDIVA_EXPORT DecimalLValue : public LValue { public: DecimalLValue(llvm::Value* data, llvm::Value* validity, llvm::Value* precision, llvm::Value* scale) diff --git a/cpp/src/gandiva/native_function.h b/cpp/src/gandiva/native_function.h index 5b130a9313c5b..82714c7de9f61 100644 --- a/cpp/src/gandiva/native_function.h +++ b/cpp/src/gandiva/native_function.h @@ -23,6 +23,7 @@ #include #include "gandiva/function_signature.h" +#include "gandiva/visibility.h" namespace gandiva { @@ -37,7 +38,7 @@ enum ResultNullableType { /// \brief Holder for the mapping from a function in an expression to a /// precompiled function. -class NativeFunction { +class GANDIVA_EXPORT NativeFunction { public: // fucntion attributes. static constexpr int32_t kNeedsContext = (1 << 1); diff --git a/cpp/src/gandiva/node.h b/cpp/src/gandiva/node.h index 77cde680d1ce8..ca51123994a0e 100644 --- a/cpp/src/gandiva/node.h +++ b/cpp/src/gandiva/node.h @@ -30,12 +30,13 @@ #include "gandiva/gandiva_aliases.h" #include "gandiva/literal_holder.h" #include "gandiva/node_visitor.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief Represents a node in the expression tree. Validity and value are /// in a joined state. -class Node { +class GANDIVA_EXPORT Node { public: explicit Node(DataTypePtr return_type) : return_type_(return_type) {} @@ -53,7 +54,7 @@ class Node { }; /// \brief Node in the expression tree, representing a literal. -class LiteralNode : public Node { +class GANDIVA_EXPORT LiteralNode : public Node { public: LiteralNode(DataTypePtr type, const LiteralHolder& holder, bool is_null) : Node(type), holder_(holder), is_null_(is_null) {} @@ -95,7 +96,7 @@ class LiteralNode : public Node { }; /// \brief Node in the expression tree, representing an arrow field. -class FieldNode : public Node { +class GANDIVA_EXPORT FieldNode : public Node { public: explicit FieldNode(FieldPtr field) : Node(field->type()), field_(field) {} @@ -112,7 +113,7 @@ class FieldNode : public Node { }; /// \brief Node in the expression tree, representing a function. -class FunctionNode : public Node { +class GANDIVA_EXPORT FunctionNode : public Node { public: FunctionNode(const std::string& name, const NodeVector& children, DataTypePtr retType); @@ -154,7 +155,7 @@ inline FunctionNode::FunctionNode(const std::string& name, const NodeVector& chi } /// \brief Node in the expression tree, representing an if-else expression. -class IfNode : public Node { +class GANDIVA_EXPORT IfNode : public Node { public: IfNode(NodePtr condition, NodePtr then_node, NodePtr else_node, DataTypePtr result_type) : Node(result_type), @@ -183,7 +184,7 @@ class IfNode : public Node { }; /// \brief Node in the expression tree, representing an and/or boolean expression. -class BooleanNode : public Node { +class GANDIVA_EXPORT BooleanNode : public Node { public: enum ExprType : char { AND, OR }; diff --git a/cpp/src/gandiva/node_visitor.h b/cpp/src/gandiva/node_visitor.h index ba3645a58969f..27d05649b8ec5 100644 --- a/cpp/src/gandiva/node_visitor.h +++ b/cpp/src/gandiva/node_visitor.h @@ -23,6 +23,7 @@ #include "arrow/status.h" #include "gandiva/logging.h" +#include "gandiva/visibility.h" namespace gandiva { @@ -35,7 +36,7 @@ template class InExpressionNode; /// \brief Visitor for nodes in the expression tree. -class NodeVisitor { +class GANDIVA_EXPORT NodeVisitor { public: virtual ~NodeVisitor() = default; diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index 83183bc460dc8..5c40a6c28af80 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -30,6 +30,18 @@ set(PRECOMPILED_SRCS timestamp_arithmetic.cc ../../arrow/util/basic_decimal.cc) +if (MSVC) + # clang pretends to be a particular version of MSVC. Version 1900 is + # Visual Studio 2015, and the standard library uses C++14 features, + # so we have to use that -std version to get the IR compilation to + # work + set(PLATFORM_CLANG_OPTIONS + -std=c++14 -fms-compatibility -fms-compatibility-version=19) +else() + set(PLATFORM_CLANG_OPTIONS + -std=c++11) +endif() + # Create bitcode for each of the source files. foreach(SRC_FILE ${PRECOMPILED_SRCS}) get_filename_component(SRC_BASE ${SRC_FILE} NAME_WE) @@ -38,10 +50,13 @@ foreach(SRC_FILE ${PRECOMPILED_SRCS}) add_custom_command( OUTPUT ${BC_FILE} COMMAND ${CLANG_EXECUTABLE} + ${PLATFORM_CLANG_OPTIONS} -DGANDIVA_IR - -std=c++11 -emit-llvm + -DNDEBUG # DCHECK macros not implemented in precompiled code + -DARROW_STATIC # Do not set __declspec(dllimport) on MSVC on Arrow symbols + -DGANDIVA_STATIC # Do not set __declspec(dllimport) on MSVC on Gandiva symbols -fno-use-cxa-atexit # Workaround for unresolved __dso_handle - -O3 -c ${ABSOLUTE_SRC} -o ${BC_FILE} + -emit-llvm -O3 -c ${ABSOLUTE_SRC} -o ${BC_FILE} ${ARROW_GANDIVA_PC_CXX_FLAGS} -I${CMAKE_SOURCE_DIR}/src DEPENDS ${SRC_FILE}) @@ -64,14 +79,20 @@ function(add_precompiled_unit_test REL_TEST_NAME) set(TEST_NAME "gandiva-precompiled-${TEST_NAME}") add_executable(${TEST_NAME} ${REL_TEST_NAME} ${ARGN}) - add_dependencies(gandiva-tests ${TEST_NAME}) target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/src) target_link_libraries(${TEST_NAME} PRIVATE ${ARROW_TEST_LINK_LIBS} ${RE2_LIBRARY} ) - target_compile_definitions(${TEST_NAME} PRIVATE GANDIVA_UNIT_TEST=1) - add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) - set_property(TEST ${TEST_NAME} PROPERTY LABELS gandiva-tests {TEST_NAME}) + target_compile_definitions(${TEST_NAME} PRIVATE + GANDIVA_UNIT_TEST=1 + ARROW_STATIC + GANDIVA_STATIC) + set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}") + add_test(${TEST_NAME} ${TEST_PATH}) + set_property(TEST ${TEST_NAME} + APPEND PROPERTY + LABELS "unittest;gandiva-tests") + add_dependencies(gandiva-tests ${TEST_NAME}) endfunction(add_precompiled_unit_test REL_TEST_NAME) # testing diff --git a/cpp/src/gandiva/precompiled/epoch_time_point_test.cc b/cpp/src/gandiva/precompiled/epoch_time_point_test.cc index f489b7d748c64..32cb9e87fe2a8 100644 --- a/cpp/src/gandiva/precompiled/epoch_time_point_test.cc +++ b/cpp/src/gandiva/precompiled/epoch_time_point_test.cc @@ -15,36 +15,39 @@ // specific language governing permissions and limitations // under the License. -#include +#include #include #include "./epoch_time_point.h" +#include "gandiva/precompiled/testing.h" #include "gandiva/precompiled/types.h" -namespace gandiva { +#include "gandiva/date_utils.h" -timestamp StringToTimestamp(const char* buf) { - struct tm tm; - strptime(buf, "%Y-%m-%d %H:%M:%S", &tm); - return timegm(&tm) * 1000; // to millis -} +namespace gandiva { TEST(TestEpochTimePoint, TestTm) { auto ts = StringToTimestamp("2015-05-07 10:20:34"); EpochTimePoint tp(ts); + struct tm* tm_ptr; +#if defined(_MSC_VER) + __time64_t tsec = ts / 1000; + tm_ptr = _gmtime64(&tsec); +#else struct tm tm; time_t tsec = ts / 1000; - gmtime_r(&tsec, &tm); - - EXPECT_EQ(tp.TmYear(), tm.tm_year); - EXPECT_EQ(tp.TmMon(), tm.tm_mon); - EXPECT_EQ(tp.TmYday(), tm.tm_yday); - EXPECT_EQ(tp.TmMday(), tm.tm_mday); - EXPECT_EQ(tp.TmWday(), tm.tm_wday); - EXPECT_EQ(tp.TmHour(), tm.tm_hour); - EXPECT_EQ(tp.TmMin(), tm.tm_min); - EXPECT_EQ(tp.TmSec(), tm.tm_sec); + tm_ptr = gmtime_r(&tsec, &tm); +#endif + + EXPECT_EQ(tp.TmYear(), tm_ptr->tm_year); + EXPECT_EQ(tp.TmMon(), tm_ptr->tm_mon); + EXPECT_EQ(tp.TmYday(), tm_ptr->tm_yday); + EXPECT_EQ(tp.TmMday(), tm_ptr->tm_mday); + EXPECT_EQ(tp.TmWday(), tm_ptr->tm_wday); + EXPECT_EQ(tp.TmHour(), tm_ptr->tm_hour); + EXPECT_EQ(tp.TmMin(), tm_ptr->tm_min); + EXPECT_EQ(tp.TmSec(), tm_ptr->tm_sec); } TEST(TestEpochTimePoint, TestAddYears) { diff --git a/cpp/src/gandiva/precompiled/extended_math_ops.cc b/cpp/src/gandiva/precompiled/extended_math_ops.cc index 1b7642cc3b3e6..b17ccd8e80a84 100644 --- a/cpp/src/gandiva/precompiled/extended_math_ops.cc +++ b/cpp/src/gandiva/precompiled/extended_math_ops.cc @@ -33,30 +33,40 @@ extern "C" { INNER(float64, OUT_TYPE) // Cubic root -#define CBRT(IN_TYPE, OUT_TYPE) \ - FORCE_INLINE \ - OUT_TYPE cbrt_##IN_TYPE(IN_TYPE in) { return static_cast(cbrtl(in)); } +#define CBRT(IN_TYPE, OUT_TYPE) \ + FORCE_INLINE \ + OUT_TYPE cbrt_##IN_TYPE(IN_TYPE in) { \ + return static_cast(cbrtl(static_cast(in))); \ + } ENUMERIC_TYPES_UNARY(CBRT, float64) // Exponent -#define EXP(IN_TYPE, OUT_TYPE) \ - FORCE_INLINE \ - OUT_TYPE exp_##IN_TYPE(IN_TYPE in) { return static_cast(expl(in)); } +#define EXP(IN_TYPE, OUT_TYPE) \ + FORCE_INLINE \ + OUT_TYPE exp_##IN_TYPE(IN_TYPE in) { \ + return static_cast(expl(static_cast(in))); \ + } ENUMERIC_TYPES_UNARY(EXP, float64) // log -#define LOG(IN_TYPE, OUT_TYPE) \ - FORCE_INLINE \ - OUT_TYPE log_##IN_TYPE(IN_TYPE in) { return static_cast(logl(in)); } +#define LOG(IN_TYPE, OUT_TYPE) \ + FORCE_INLINE \ + OUT_TYPE log_##IN_TYPE(IN_TYPE in) { \ + return static_cast(logl(static_cast(in))); \ + } ENUMERIC_TYPES_UNARY(LOG, float64) // log base 10 -#define LOG10(IN_TYPE, OUT_TYPE) \ - FORCE_INLINE \ - OUT_TYPE log10_##IN_TYPE(IN_TYPE in) { return static_cast(log10l(in)); } +#define LOG10(IN_TYPE, OUT_TYPE) \ + FORCE_INLINE \ + OUT_TYPE log10_##IN_TYPE(IN_TYPE in) { \ + return static_cast(log10l(static_cast(in))); \ + } + +#define LOGL(VALUE) static_cast(logl(static_cast(VALUE))) ENUMERIC_TYPES_UNARY(LOG10, float64) @@ -74,12 +84,12 @@ void set_error_for_logbase(int64_t execution_context, double base) { #define LOG_WITH_BASE(IN_TYPE1, IN_TYPE2, OUT_TYPE) \ FORCE_INLINE \ OUT_TYPE log_##IN_TYPE1##_##IN_TYPE2(int64 context, IN_TYPE1 base, IN_TYPE2 value) { \ - OUT_TYPE log_of_base = static_cast(logl(base)); \ + OUT_TYPE log_of_base = LOGL(base); \ if (log_of_base == 0) { \ set_error_for_logbase(context, static_cast(base)); \ return 0; \ } \ - return static_cast(logl(value) / logl(base)); \ + return LOGL(value) / LOGL(base); \ } LOG_WITH_BASE(int32, int32, float64) diff --git a/cpp/src/gandiva/precompiled/testing.h b/cpp/src/gandiva/precompiled/testing.h new file mode 100644 index 0000000000000..3214eec6f0494 --- /dev/null +++ b/cpp/src/gandiva/precompiled/testing.h @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "arrow/util/logging.h" + +#include "gandiva/date_utils.h" +#include "gandiva/precompiled/types.h" + +namespace gandiva { + +timestamp StringToTimestamp(const char* buf) { + int64_t out = 0; + DCHECK(internal::ParseTimestamp(buf, "%Y-%m-%d %H:%M:%S", false, &out)); + return out * 1000; +} + +} // namespace gandiva diff --git a/cpp/src/gandiva/precompiled/time_test.cc b/cpp/src/gandiva/precompiled/time_test.cc index 36ba9b308e621..b8f80698200c0 100644 --- a/cpp/src/gandiva/precompiled/time_test.cc +++ b/cpp/src/gandiva/precompiled/time_test.cc @@ -18,16 +18,11 @@ #include #include #include "../execution_context.h" +#include "gandiva/precompiled/testing.h" #include "gandiva/precompiled/types.h" namespace gandiva { -timestamp StringToTimestamp(const char* buf) { - struct tm tm; - strptime(buf, "%Y-%m-%d %H:%M:%S", &tm); - return timegm(&tm) * 1000; // to millis -} - TEST(TestTime, TestCastDate) { ExecutionContext context; int64_t context_ptr = reinterpret_cast(&context); diff --git a/cpp/src/gandiva/projector.cc b/cpp/src/gandiva/projector.cc index 8fc5b8c446927..7950fc7b8d149 100644 --- a/cpp/src/gandiva/projector.cc +++ b/cpp/src/gandiva/projector.cc @@ -36,6 +36,8 @@ Projector::Projector(std::unique_ptr llvm_generator, SchemaPtr sc output_fields_(output_fields), configuration_(configuration) {} +Projector::~Projector() {} + Status Projector::Make(SchemaPtr schema, const ExpressionVector& exprs, std::shared_ptr* projector) { return Projector::Make(schema, exprs, ConfigurationBuilder::DefaultConfiguration(), diff --git a/cpp/src/gandiva/projector.h b/cpp/src/gandiva/projector.h index c9d727164735e..58bac78a4068c 100644 --- a/cpp/src/gandiva/projector.h +++ b/cpp/src/gandiva/projector.h @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -#ifndef GANDIVA_EXPR_PROJECTOR_H -#define GANDIVA_EXPR_PROJECTOR_H +#pragma once #include #include @@ -28,6 +27,7 @@ #include "gandiva/arrow.h" #include "gandiva/configuration.h" #include "gandiva/expression.h" +#include "gandiva/visibility.h" namespace gandiva { @@ -37,8 +37,12 @@ class LLVMGenerator; /// /// A projector is built for a specific schema and vector of expressions. /// Once the projector is built, it can be used to evaluate many row batches. -class Projector { +class GANDIVA_EXPORT Projector { public: + // Inline dtor will attempt to resolve the destructor for + // LLVMGenerator on MSVC, so we compile the dtor in the object code + ~Projector(); + /// Build a default projector for the given schema to evaluate /// the vector of expressions. /// @@ -99,5 +103,3 @@ class Projector { }; } // namespace gandiva - -#endif // GANDIVA_EXPR_PROJECTOR_H diff --git a/cpp/src/gandiva/regex_util.h b/cpp/src/gandiva/regex_util.h index 6a22af2b9cac3..7ea7060c483f8 100644 --- a/cpp/src/gandiva/regex_util.h +++ b/cpp/src/gandiva/regex_util.h @@ -23,11 +23,12 @@ #include #include "gandiva/arrow.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief Utility class for converting sql patterns to pcre patterns. -class RegexUtil { +class GANDIVA_EXPORT RegexUtil { public: // Convert an sql pattern to a pcre pattern static Status SqlLikePatternToPcre(const std::string& like_pattern, char escape_char, diff --git a/cpp/src/gandiva/selection_vector.cc b/cpp/src/gandiva/selection_vector.cc index f89b80c2b510f..e643cece8a2c3 100644 --- a/cpp/src/gandiva/selection_vector.cc +++ b/cpp/src/gandiva/selection_vector.cc @@ -22,6 +22,8 @@ #include #include +#include "arrow/util/bit-util.h" + #include "gandiva/selection_vector_impl.h" namespace gandiva { @@ -48,8 +50,18 @@ Status SelectionVector::PopulateFromBitMap(const uint8_t* bitmap, int64_t bitmap uint64_t current_word = bitmap_64[bitmap_idx]; while (current_word != 0) { +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4146) +#endif + // MSVC warns about negating an unsigned type. We suppress it for now uint64_t highest_only = current_word & -current_word; - int pos_in_word = __builtin_ctzl(highest_only); + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + + int pos_in_word = arrow::BitUtil::CountTrailingZeros(highest_only); int64_t pos_in_bitmap = bitmap_idx * 64 + pos_in_word; if (pos_in_bitmap > max_bitmap_index) { diff --git a/cpp/src/gandiva/selection_vector.h b/cpp/src/gandiva/selection_vector.h index dcd2f6bbb7f4c..2e9941781d01e 100644 --- a/cpp/src/gandiva/selection_vector.h +++ b/cpp/src/gandiva/selection_vector.h @@ -24,12 +24,13 @@ #include "gandiva/arrow.h" #include "gandiva/logging.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief Selection Vector : vector of indices in a row-batch for a selection, /// backed by an arrow-array. -class SelectionVector { +class GANDIVA_EXPORT SelectionVector { public: virtual ~SelectionVector() = default; diff --git a/cpp/src/gandiva/tests/date_time_test.cc b/cpp/src/gandiva/tests/date_time_test.cc index 643b8c8dda3ce..7867a9500fddb 100644 --- a/cpp/src/gandiva/tests/date_time_test.cc +++ b/cpp/src/gandiva/tests/date_time_test.cc @@ -57,7 +57,7 @@ int64_t MillisSince(time_t base_line, int32_t yy, int32_t mm, int32_t dd, int32_ given_ts.tm_min = min; given_ts.tm_sec = sec; - return (lround(difftime(mktime(&given_ts), base_line)) * 1000 + millis); + return (static_cast(difftime(mktime(&given_ts), base_line)) * 1000 + millis); } TEST_F(TestProjector, TestIsNull) { diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 5c32f5024dba0..ba0e63292f4ab 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -86,10 +86,14 @@ TEST_F(TestProjector, TestProjectCache) { EXPECT_EQ(cached_projector, projector); // if configuration is different, should return a new projector. + + // build a new path by replacing the first '/' with '//' + std::string alt_path(GANDIVA_BYTE_COMPILE_FILE_PATH); + auto pos = alt_path.find('/', 0); + EXPECT_NE(pos, std::string::npos); + alt_path.replace(pos, 1, "//"); auto other_configuration = - ConfigurationBuilder() - .set_byte_code_file_path("/" + std::string(GANDIVA_BYTE_COMPILE_FILE_PATH)) - .build(); + ConfigurationBuilder().set_byte_code_file_path(alt_path).build(); std::shared_ptr should_be_new_projector2; status = Projector::Make(schema, {sum_expr, sub_expr}, other_configuration, &should_be_new_projector2); diff --git a/cpp/src/gandiva/to_date_holder.cc b/cpp/src/gandiva/to_date_holder.cc index 6e02a6a0d10db..824654f44a6e4 100644 --- a/cpp/src/gandiva/to_date_holder.cc +++ b/cpp/src/gandiva/to_date_holder.cc @@ -18,7 +18,7 @@ #include #include -#include "arrow/vendored/datetime/date.h" +#include "arrow/vendored/datetime.h" #include "gandiva/date_utils.h" #include "gandiva/execution_context.h" @@ -64,8 +64,7 @@ Status ToDateHolder::Make(const FunctionNode& node, Status ToDateHolder::Make(const std::string& sql_pattern, int32_t suppress_errors, std::shared_ptr* holder) { std::shared_ptr transformed_pattern; - Status status = DateUtils::ToInternalFormat(sql_pattern, &transformed_pattern); - ARROW_RETURN_NOT_OK(status); + ARROW_RETURN_NOT_OK(DateUtils::ToInternalFormat(sql_pattern, &transformed_pattern)); auto lholder = std::shared_ptr( new ToDateHolder(*(transformed_pattern.get()), suppress_errors)); *holder = lholder; @@ -82,22 +81,14 @@ int64_t ToDateHolder::operator()(ExecutionContext* context, const std::string& d // Issues // 1. processes date that do not match the format. // 2. does not process time in format +08:00 (or) id. - struct tm result = {}; - char* ret = strptime(data.c_str(), pattern_.c_str(), &result); - if (ret == nullptr) { + int64_t seconds_since_epoch = 0; + if (!internal::ParseTimestamp(data.c_str(), pattern_.c_str(), true, + &seconds_since_epoch)) { return_error(context, data); return 0; } + *out_valid = true; - // ignore the time part - arrow::util::date::sys_seconds secs = - arrow::util::date::sys_days(arrow::util::date::year(result.tm_year + 1900) / - (result.tm_mon + 1) / result.tm_mday); - int64_t seconds_since_epoch = secs.time_since_epoch().count(); - if (seconds_since_epoch == 0) { - return_error(context, data); - return 0; - } return seconds_since_epoch * 1000; } diff --git a/cpp/src/gandiva/to_date_holder.h b/cpp/src/gandiva/to_date_holder.h index 91133cc5269d8..c0c5afb8b31cd 100644 --- a/cpp/src/gandiva/to_date_holder.h +++ b/cpp/src/gandiva/to_date_holder.h @@ -27,11 +27,12 @@ #include "gandiva/execution_context.h" #include "gandiva/function_holder.h" #include "gandiva/node.h" +#include "gandiva/visibility.h" namespace gandiva { /// Function Holder for SQL 'to_date' -class ToDateHolder : public FunctionHolder { +class GANDIVA_EXPORT ToDateHolder : public FunctionHolder { public: ~ToDateHolder() override = default; diff --git a/cpp/src/gandiva/to_date_holder_test.cc b/cpp/src/gandiva/to_date_holder_test.cc index 2a207b2ad7742..0effffb0ddb7c 100644 --- a/cpp/src/gandiva/to_date_holder_test.cc +++ b/cpp/src/gandiva/to_date_holder_test.cc @@ -18,6 +18,8 @@ #include #include +#include "arrow/test-util.h" + #include "gandiva/execution_context.h" #include "gandiva/to_date_holder.h" #include "precompiled/epoch_time_point.h" @@ -37,57 +39,68 @@ class TestToDateHolder : public ::testing::Test { return FunctionNode("to_date_utf8_utf8_int32", {field, pattern_node, suppres_error_node}, arrow::int64()); } + + protected: + ExecutionContext execution_context_; }; TEST_F(TestToDateHolder, TestSimpleDateTime) { std::shared_ptr to_date_holder; + ASSERT_OK(ToDateHolder::Make("YYYY-MM-DD HH:MI:SS", 1, &to_date_holder)); - auto status = ToDateHolder::Make("YYYY-MM-DD HH:MI:SS", 1, &to_date_holder); - EXPECT_EQ(status.ok(), true) << status.message(); - ExecutionContext execution_context; auto& to_date = *to_date_holder; bool out_valid; int64_t millis_since_epoch = - to_date(&execution_context, "1986-12-01 01:01:01", true, &out_valid); + to_date(&execution_context_, "1986-12-01 01:01:01", true, &out_valid); EXPECT_EQ(millis_since_epoch, 533779200000); millis_since_epoch = - to_date(&execution_context, "1986-12-01 01:01:01.11", true, &out_valid); + to_date(&execution_context_, "1986-12-01 01:01:01.11", true, &out_valid); EXPECT_EQ(millis_since_epoch, 533779200000); millis_since_epoch = - to_date(&execution_context, "1986-12-01 01:01:01 +0800", true, &out_valid); + to_date(&execution_context_, "1986-12-01 01:01:01 +0800", true, &out_valid); EXPECT_EQ(millis_since_epoch, 533779200000); +#if 0 + // TODO : this fails parsing with date::parse and strptime on linux + millis_since_epoch = + to_date(&execution_context_, "1886-12-01 00:00:00", true, &out_valid); + EXPECT_EQ(out_valid, true); + EXPECT_EQ(millis_since_epoch, -2621894400000); +#endif + millis_since_epoch = - to_date(&execution_context, "1986-12-11 01:30:00", true, &out_valid); + to_date(&execution_context_, "1886-12-01 01:01:01", true, &out_valid); + EXPECT_EQ(millis_since_epoch, -2621894400000); + + millis_since_epoch = + to_date(&execution_context_, "1986-12-11 01:30:00", true, &out_valid); EXPECT_EQ(millis_since_epoch, 534643200000); } TEST_F(TestToDateHolder, TestSimpleDate) { std::shared_ptr to_date_holder; + ASSERT_OK(ToDateHolder::Make("YYYY-MM-DD", 1, &to_date_holder)); - auto status = ToDateHolder::Make("YYYY-MM-DD", 1, &to_date_holder); - EXPECT_EQ(status.ok(), true) << status.message(); - ExecutionContext execution_context; auto& to_date = *to_date_holder; bool out_valid; int64_t millis_since_epoch = - to_date(&execution_context, "1986-12-01", true, &out_valid); + to_date(&execution_context_, "1986-12-01", true, &out_valid); EXPECT_EQ(millis_since_epoch, 533779200000); - millis_since_epoch = to_date(&execution_context, "1986-12-1", true, &out_valid); + millis_since_epoch = to_date(&execution_context_, "1986-12-1", true, &out_valid); EXPECT_EQ(millis_since_epoch, 533779200000); - millis_since_epoch = to_date(&execution_context, "1886-12-1", true, &out_valid); + millis_since_epoch = to_date(&execution_context_, "1886-12-1", true, &out_valid); EXPECT_EQ(millis_since_epoch, -2621894400000); - millis_since_epoch = to_date(&execution_context, "2012-12-1", true, &out_valid); + millis_since_epoch = to_date(&execution_context_, "2012-12-1", true, &out_valid); EXPECT_EQ(millis_since_epoch, 1354320000000); // wrong month. should return 0 since we are suppresing errors. millis_since_epoch = - to_date(&execution_context, "1986-21-01 01:01:01 +0800", true, &out_valid); + to_date(&execution_context_, "1986-21-01 01:01:01 +0800", true, &out_valid); EXPECT_EQ(millis_since_epoch, 0); } @@ -96,22 +109,22 @@ TEST_F(TestToDateHolder, TestSimpleDateTimeError) { auto status = ToDateHolder::Make("YYYY-MM-DD HH:MI:SS", 0, &to_date_holder); EXPECT_EQ(status.ok(), true) << status.message(); - ExecutionContext execution_context; auto& to_date = *to_date_holder; bool out_valid; int64_t millis_since_epoch = - to_date(&execution_context, "1986-21-01 01:01:01 +0800", true, &out_valid); + to_date(&execution_context_, "1986-01-40 01:01:01 +0800", true, &out_valid); + EXPECT_EQ(0, millis_since_epoch); std::string expected_error = - "Error parsing value 1986-21-01 01:01:01 +0800 for given format"; - EXPECT_TRUE(execution_context.get_error().find(expected_error) != std::string::npos) + "Error parsing value 1986-01-40 01:01:01 +0800 for given format"; + EXPECT_TRUE(execution_context_.get_error().find(expected_error) != std::string::npos) << status.message(); // not valid should not return error - execution_context.Reset(); - millis_since_epoch = to_date(&execution_context, "nullptr", false, &out_valid); + execution_context_.Reset(); + millis_since_epoch = to_date(&execution_context_, "nullptr", false, &out_valid); EXPECT_EQ(millis_since_epoch, 0); - EXPECT_TRUE(execution_context.has_error() == false); + EXPECT_TRUE(execution_context_.has_error() == false); } TEST_F(TestToDateHolder, TestSimpleDateTimeMakeError) { diff --git a/cpp/src/gandiva/tree_expr_builder.h b/cpp/src/gandiva/tree_expr_builder.h index 3d60b5b96168d..4b2789af04c04 100644 --- a/cpp/src/gandiva/tree_expr_builder.h +++ b/cpp/src/gandiva/tree_expr_builder.h @@ -27,11 +27,12 @@ #include "gandiva/condition.h" #include "gandiva/decimal_scalar.h" #include "gandiva/expression.h" +#include "gandiva/visibility.h" namespace gandiva { /// \brief Tree Builder for a nested expression. -class TreeExprBuilder { +class GANDIVA_EXPORT TreeExprBuilder { public: /// \brief create a node on a literal. static NodePtr MakeLiteral(bool value); diff --git a/cpp/src/gandiva/value_validity_pair.h b/cpp/src/gandiva/value_validity_pair.h index 1bcd5d6a4bfd2..0de525d97040f 100644 --- a/cpp/src/gandiva/value_validity_pair.h +++ b/cpp/src/gandiva/value_validity_pair.h @@ -21,11 +21,12 @@ #include #include "gandiva/gandiva_aliases.h" +#include "gandiva/visibility.h" namespace gandiva { /// Pair of vector/validities generated after decomposing an expression tree/subtree. -class ValueValidityPair { +class GANDIVA_EXPORT ValueValidityPair { public: ValueValidityPair(const DexVector& validity_exprs, DexPtr value_expr) : validity_exprs_(validity_exprs), value_expr_(value_expr) {} diff --git a/cpp/src/gandiva/visibility.h b/cpp/src/gandiva/visibility.h new file mode 100644 index 0000000000000..450b3056b2ec0 --- /dev/null +++ b/cpp/src/gandiva/visibility.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(_WIN32) || defined(__CYGWIN__) +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4251) +#else +#pragma GCC diagnostic ignored "-Wattributes" +#endif + +#ifdef GANDIVA_STATIC +#define GANDIVA_EXPORT +#elif defined(GANDIVA_EXPORTING) +#define GANDIVA_EXPORT __declspec(dllexport) +#else +#define GANDIVA_EXPORT __declspec(dllimport) +#endif + +#define GANDIVA_NO_EXPORT +#else // Not Windows +#ifndef GANDIVA_EXPORT +#define GANDIVA_EXPORT __attribute__((visibility("default"))) +#endif +#ifndef GANDIVA_NO_EXPORT +#define GANDIVA_NO_EXPORT __attribute__((visibility("hidden"))) +#endif +#endif // Non-Windows + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 00cd31e6df969..e62a37b082407 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -27,7 +27,7 @@ BOOST_VERSION=1.67.0 BROTLI_VERSION=v0.6.0 CARES_VERSION=1.15.0 DOUBLE_CONVERSION_VERSION=v3.1.1 -FLATBUFFERS_VERSION=02a7807dd8d26f5668ffbbec0360dc107bbfabd5 +FLATBUFFERS_VERSION=v1.10.0 GBENCHMARK_VERSION=v1.4.1 GFLAGS_VERSION=v2.2.0 GLOG_VERSION=v0.3.5 From 76da3cd9118298d39dcad4b8ed420371df5943c4 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 1 Feb 2019 06:08:56 +0100 Subject: [PATCH 16/20] [Python] Use 'latest' manylinux1 build image Master build of the docker image has passed, so use this one again. --- .travis.yml | 2 +- ci/travis_script_manylinux.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c037cd5d914e6..02ce11de9b121 100644 --- a/.travis.yml +++ b/.travis.yml @@ -225,7 +225,7 @@ matrix: - name: "[manylinux1] Python" language: cpp before_script: - - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-4424; fi + - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest; fi script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi - name: "Java w/ OpenJDK 8" diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index a7d82cc76c2bc..588d0f9a7bbb6 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -21,7 +21,7 @@ set -ex pushd python/manylinux1 -docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io -v $PWD/../../:/arrow quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-4424 /io/build_arrow.sh +docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io -v $PWD/../../:/arrow quay.io/xhochy/arrow_manylinux1_x86_64_base:latest /io/build_arrow.sh # Testing for https://issues.apache.org/jira/browse/ARROW-2657 # These tests cannot be run inside of the docker container, since TensorFlow From caa3a92d9f810fae35cea6f12f3af5e9c2444e60 Mon Sep 17 00:00:00 2001 From: Kenta Murata Date: Thu, 31 Jan 2019 23:10:09 -0600 Subject: [PATCH 17/20] ARROW-4397: [C++] Add dim_names in Tensor and SparseTensor Along with [ARROW-4388](https://issues.apache.org/jira/browse/ARROW-4388), it would be useful to introduce dim_names in Tensor and SparseTensor of C++ library. Author: Kenta Murata Closes #3507 from mrkn/cpp_tensor_dim_names and squashes the following commits: f529ee27 Add dim_names in Tensor and SparseTensor --- cpp/src/arrow/sparse_tensor-test.cc | 8 ++++++++ cpp/src/arrow/sparse_tensor.h | 1 + cpp/src/arrow/tensor-test.cc | 3 +++ cpp/src/arrow/tensor.h | 1 + 4 files changed, 13 insertions(+) diff --git a/cpp/src/arrow/sparse_tensor-test.cc b/cpp/src/arrow/sparse_tensor-test.cc index ed51f03f88841..0a3e98611ba7c 100644 --- a/cpp/src/arrow/sparse_tensor-test.cc +++ b/cpp/src/arrow/sparse_tensor-test.cc @@ -60,10 +60,12 @@ TEST(TestSparseCOOTensor, CreationEmptyTensor) { ASSERT_EQ(24, st1.size()); ASSERT_EQ(24, st2.size()); + ASSERT_EQ(std::vector({"foo", "bar", "baz"}), st2.dim_names()); ASSERT_EQ("foo", st2.dim_name(0)); ASSERT_EQ("bar", st2.dim_name(1)); ASSERT_EQ("baz", st2.dim_name(2)); + ASSERT_EQ(std::vector({}), st1.dim_names()); ASSERT_EQ("", st1.dim_name(0)); ASSERT_EQ("", st1.dim_name(1)); ASSERT_EQ("", st1.dim_name(2)); @@ -85,10 +87,12 @@ TEST(TestSparseCOOTensor, CreationFromNumericTensor) { ASSERT_EQ(12, st1.non_zero_length()); ASSERT_TRUE(st1.is_mutable()); + ASSERT_EQ(std::vector({"foo", "bar", "baz"}), st2.dim_names()); ASSERT_EQ("foo", st2.dim_name(0)); ASSERT_EQ("bar", st2.dim_name(1)); ASSERT_EQ("baz", st2.dim_name(2)); + ASSERT_EQ(std::vector({}), st1.dim_names()); ASSERT_EQ("", st1.dim_name(0)); ASSERT_EQ("", st1.dim_name(1)); ASSERT_EQ("", st1.dim_name(2)); @@ -124,10 +128,12 @@ TEST(TestSparseCOOTensor, CreationFromTensor) { ASSERT_EQ(12, st1.non_zero_length()); ASSERT_TRUE(st1.is_mutable()); + ASSERT_EQ(std::vector({"foo", "bar", "baz"}), st2.dim_names()); ASSERT_EQ("foo", st2.dim_name(0)); ASSERT_EQ("bar", st2.dim_name(1)); ASSERT_EQ("baz", st2.dim_name(2)); + ASSERT_EQ(std::vector({}), st1.dim_names()); ASSERT_EQ("", st1.dim_name(0)); ASSERT_EQ("", st1.dim_name(1)); ASSERT_EQ("", st1.dim_name(2)); @@ -192,10 +198,12 @@ TEST(TestSparseCSRMatrix, CreationFromNumericTensor2D) { ASSERT_EQ(12, st1.non_zero_length()); ASSERT_TRUE(st1.is_mutable()); + ASSERT_EQ(std::vector({"foo", "bar", "baz"}), st2.dim_names()); ASSERT_EQ("foo", st2.dim_name(0)); ASSERT_EQ("bar", st2.dim_name(1)); ASSERT_EQ("baz", st2.dim_name(2)); + ASSERT_EQ(std::vector({}), st1.dim_names()); ASSERT_EQ("", st1.dim_name(0)); ASSERT_EQ("", st1.dim_name(1)); ASSERT_EQ("", st1.dim_name(2)); diff --git a/cpp/src/arrow/sparse_tensor.h b/cpp/src/arrow/sparse_tensor.h index c7693d2ec9579..ded3a6d9bf8e3 100644 --- a/cpp/src/arrow/sparse_tensor.h +++ b/cpp/src/arrow/sparse_tensor.h @@ -135,6 +135,7 @@ class ARROW_EXPORT SparseTensor { int ndim() const { return static_cast(shape_.size()); } + const std::vector& dim_names() const { return dim_names_; } const std::string& dim_name(int i) const; /// Total number of value cells in the sparse tensor diff --git a/cpp/src/arrow/tensor-test.cc b/cpp/src/arrow/tensor-test.cc index af20aed0d6ec1..11ea7c2a0ca69 100644 --- a/cpp/src/arrow/tensor-test.cc +++ b/cpp/src/arrow/tensor-test.cc @@ -66,8 +66,11 @@ TEST(TestTensor, BasicCtors) { ASSERT_EQ(strides, t1.strides()); ASSERT_EQ(strides, t2.strides()); + ASSERT_EQ(std::vector({"foo", "bar"}), t3.dim_names()); ASSERT_EQ("foo", t3.dim_name(0)); ASSERT_EQ("bar", t3.dim_name(1)); + + ASSERT_EQ(std::vector({}), t1.dim_names()); ASSERT_EQ("", t1.dim_name(0)); ASSERT_EQ("", t1.dim_name(1)); } diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h index 445a81f2cf24c..fb2093b915730 100644 --- a/cpp/src/arrow/tensor.h +++ b/cpp/src/arrow/tensor.h @@ -82,6 +82,7 @@ class ARROW_EXPORT Tensor { int ndim() const { return static_cast(shape_.size()); } + const std::vector& dim_names() const { return dim_names_; } const std::string& dim_name(int i) const; /// Total number of value cells in the tensor From 8ab0212de91d4b2179e14a6d451c1aa3a440c296 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Fri, 1 Feb 2019 10:03:36 +0100 Subject: [PATCH 18/20] ARROW-4277: [C++] Add gmock to the toolchain To some extent this was an educated guess on CMake settings (with a lot of copy and pasting, so I expect things could be done slightly better). Author: Micah Kornfield Closes #3416 from emkornfield/gmock and squashes the following commits: d6b143fe try to fix mac 9f8854eb revert version bump for gtest and append GMOCK directory before other includes e3b3a32c try to use vendored gmock on trusty 4c2425bf reorder gtest library for trusty? 367abc9f Can more typos 2cf4a268 Remove force found and add more messaging 43f47af2 remove unused GMOCK_HOME 0a738b9e remove prefix 7c78244a force gmock true 218506e3 fix condition 84bab120 fix environment variable f7f23a0d remove duplicate gmock 0c0ff927 fix some typos 812f4142 update gmock output description in gmock modules 626621a1 cleanup bad suffixes and use gtest_roots 4d69be85 fix cmake gmock message and format 7c05e73d update cmake recipe with copy/paste bug fixes be8020a4 add gmock to conda package list e4d081ab add mock f2c47681 replace DummyBinaryKernel 3c51702c add many linux build 911f093a Add gmock (mostly guesswork and copy/paste) --- ci/conda_env_cpp.yml | 1 + cpp/CMakeLists.txt | 10 ++- cpp/cmake_modules/FindGTest.cmake | 93 ++++++++++++++++++++- cpp/cmake_modules/ThirdpartyToolchain.cmake | 47 ++++++++++- cpp/src/arrow/compute/compute-test.cc | 9 +- cpp/src/arrow/compute/test-util.h | 13 +++ python/manylinux1/scripts/build_gtest.sh | 4 +- 7 files changed, 158 insertions(+), 19 deletions(-) diff --git a/ci/conda_env_cpp.yml b/ci/conda_env_cpp.yml index 87523b3fdd611..3d50b210ea68d 100644 --- a/ci/conda_env_cpp.yml +++ b/ci/conda_env_cpp.yml @@ -25,6 +25,7 @@ double-conversion flatbuffers gflags glog +gmock gtest libprotobuf lz4-c diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9cdbe7dc801fb..e0dbcd305e92e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -800,8 +800,9 @@ set(ARROW_TEST_STATIC_LINK_LIBS arrow_testing_static arrow_static ${ARROW_LINK_LIBS} - ${GTEST_MAIN_LIBRARY} - ${GTEST_LIBRARY}) + ${GTEST_LIBRARY} + ${GMOCK_MAIN_LIBRARY} + ${GMOCK_LIBRARY}) set(ARROW_TEST_SHARED_LINK_LIBS arrow_testing_shared @@ -811,8 +812,9 @@ set(ARROW_TEST_SHARED_LINK_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_REGEX_LIBRARY} - ${GTEST_MAIN_LIBRARY} - ${GTEST_LIBRARY}) + ${GTEST_LIBRARY} + ${GMOCK_MAIN_LIBRARY} + ${GMOCK_LIBRARY}) if(NOT MSVC) set(ARROW_TEST_SHARED_LINK_LIBS diff --git a/cpp/cmake_modules/FindGTest.cmake b/cpp/cmake_modules/FindGTest.cmake index c7496c6a3b9f1..6ddb14aa6fb60 100644 --- a/cpp/cmake_modules/FindGTest.cmake +++ b/cpp/cmake_modules/FindGTest.cmake @@ -21,8 +21,8 @@ # to be set before calling find_package: # # GTest_HOME - When set, this path is inspected instead of standard library -# locations as the root of the GTest installation. -# The environment variable GTEST_HOME overrides this veriable. +# locations as the root of the GTest/Gmock installation. +# The environment variable GTEST_HOME overrides this variable. # # This module defines # GTEST_INCLUDE_DIR, directory containing headers @@ -32,6 +32,14 @@ # GTEST_SHARED_LIB, path to libgtest's shared library # GTEST_SHARED_MAIN_LIB, path to libgtest_main's shared library # GTEST_FOUND, whether gtest has been found +# +# GMOCK_INCLUDE_DIR, directory containing headers +# GMOCK_LIBS, directory containing gmock libraries +# GMOCK_STATIC_LIB, path to libgmock.a +# GMOCK_STATIC_MAIN_LIB, path to libgmock_main.a +# GMOCK_SHARED_LIB, path to libgmock's shared library +# GMOCK_SHARED_MAIN_LIB, path to libgmock_main's shared library +# GMOCK_FOUND, whether gmock has been found if( NOT "${GTEST_HOME}" STREQUAL "") file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path ) @@ -48,6 +56,15 @@ set(GTEST_SHARED_LIB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}gtest${CMAKE_SHARED_LIBRARY_SUFFIX}) set(GTEST_MAIN_SHARED_LIB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}gtest_main${CMAKE_SHARED_LIBRARY_SUFFIX}) +set(GMOCK_STATIC_LIB_NAME + ${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}) +set(GMOCK_MAIN_STATIC_LIB_NAME + ${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_STATIC_LIBRARY_SUFFIX}) +set(GMOCK_SHARED_LIB_NAME + ${CMAKE_SHARED_LIBRARY_PREFIX}gmock${CMAKE_SHARED_LIBRARY_SUFFIX}) +set(GMOCK_MAIN_SHARED_LIB_NAME + ${CMAKE_SHARED_LIBRARY_PREFIX}gmock_main${CMAKE_SHARED_LIBRARY_SUFFIX}) + # Try the parameterized roots, if they exist if(_gtest_roots) @@ -78,6 +95,36 @@ else() find_library(GTEST_MAIN_SHARED_LIB NAMES ${GTEST_MAIN_SHARED_LIB_NAME}) endif() +# gmock +# Try the parameterized roots, if they exist (reuse gtest because the +# libraries should be built together). +if(_gtest_roots) + find_path(GMOCK_INCLUDE_DIR NAMES gmock/gmock.h + PATHS ${_gtest_roots} NO_DEFAULT_PATH + PATH_SUFFIXES "include") + set(lib_dirs + "lib/${CMAKE_LIBRARY_ARCHITECTURE}" + "lib64" + "lib") + find_library(GMOCK_STATIC_LIB NAMES ${GMOCK_STATIC_LIB_NAME} + PATHS ${_gtest_roots} NO_DEFAULT_PATH + PATH_SUFFIXES ${lib_dirs}) + find_library(GMOCK_MAIN_STATIC_LIB NAMES ${GMOCK_MAIN_STATIC_LIB_NAME} + PATHS ${_gtest_roots} NO_DEFAULT_PATH + PATH_SUFFIXES ${lib_dirs}) + find_library(GMOCK_SHARED_LIB NAMES ${GMOCK_SHARED_LIB_NAME} + PATHS ${_gtest_roots} NO_DEFAULT_PATH + PATH_SUFFIXES ${lib_dirs}) + find_library(GMOCK_MAIN_SHARED_LIB NAMES ${GMOCK_MAIN_SHARED_LIB_NAME} + PATHS ${_gtest_roots} NO_DEFAULT_PATH + PATH_SUFFIXES ${lib_dirs}) +else() + find_path(GMOCK_INCLUDE_DIR NAMES gmock/gmock.h) + find_library(GMOCK_STATIC_LIB NAMES ${GMOCK_STATIC_LIB_NAME}) + find_library(GMOCK_MAIN_STATIC_LIB NAMES ${GMOCK_MAIN_STATIC_LIB_NAME}) + find_library(GMOCK_SHARED_LIB NAMES ${GMOCK_SHARED_LIB_NAME}) + find_library(GMOCK_MAIN_SHARED_LIB NAMES ${GMOCK_MAIN_SHARED_LIB_NAME}) +endif() if(GTEST_INCLUDE_DIR AND (GTEST_STATIC_LIB AND GTEST_MAIN_STATIC_LIB) OR @@ -87,6 +134,14 @@ else() set(GTEST_FOUND FALSE) endif() +if(GMOCK_INCLUDE_DIR AND + (GMOCK_STATIC_LIB AND GMOCK_MAIN_STATIC_LIB) OR + (GMOCK_SHARED_LIB AND GMOCK_MAIN_SHARED_LIB)) + set(GMOCK_FOUND TRUE) +else() + set(GMOCK_FOUND FALSE) +endif() + if (GTEST_FOUND) if (NOT GTest_FIND_QUIETLY) message(STATUS "Found the GTest library:") @@ -111,6 +166,33 @@ else () endif () endif () +if (GMOCK_FOUND) + if (NOT GTest_FIND_QUIETLY) + message(STATUS "Found the Gmock library:") + message(STATUS "GMOCK_STATIC_LIB: ${GMOCK_STATIC_LIB}") + message(STATUS "GMOCK_MAIN_STATIC_LIB: ${GMOCK_MAIN_STATIC_LIB}") + message(STATUS "GMOCK_SHARED_LIB: ${GMOCK_SHARED_LIB}") + message(STATUS "GMOCK_MAIN_SHARED_LIB: ${GMOCK_MAIN_SHARED_LIB}") + endif () +else () + # Reuse Gtest quietly and required flags because they should be found + # in tandem. + if (NOT GTest_FIND_QUIETLY) + set(GMOCK_ERR_MSG "Could not find the GMock library. Looked in ") + if ( _gtest_roots ) + set(GTEST_ERR_MSG "${GMOCK_ERR_MSG} in ${_gtest_roots}.") + else () + set(GTEST_ERR_MSG "${GMOCK_ERR_MSG} system search paths.") + endif () + if (GTest_FIND_REQUIRED) + message(FATAL_ERROR "${GMOCK_ERR_MSG}") + else (GTest_FIND_REQUIRED) + message(STATUS "${GMOCK_ERR_MSG}") + endif (GTest_FIND_REQUIRED) + endif () +endif () + + mark_as_advanced( GTEST_INCLUDE_DIR GTEST_LIBS @@ -118,4 +200,11 @@ mark_as_advanced( GTEST_MAIN_STATIC_LIB GTEST_SHARED_LIB GTEST_MAIN_SHARED_LIB + + GMOCK_INCLUDE_DIR + GMOCK_LIBS + GMOCK_STATIC_LIB + GMOCK_MAIN_STATIC_LIB + GMOCK_SHARED_LIB + GMOCK_MAIN_SHARED_LIB ) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index fedeed489b256..ff2252528fdf3 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -30,6 +30,7 @@ set(ARROW_RE2_LINKAGE "static" CACHE STRING set(THIRDPARTY_DIR "${arrow_SOURCE_DIR}/thirdparty") + if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "") set(BROTLI_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") set(BZ2_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") @@ -39,9 +40,21 @@ if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "") set(GFLAGS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") set(GLOG_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") set(GRPC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - # Using gtest from the toolchain breaks AppVeyor builds + # Using gtest from the toolchain breaks AppVeyor and + # trusty builds if (NOT MSVC) - set(GTEST_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") + if (APPLE) + set(GTEST_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") + else() + #linux + execute_process(COMMAND lsb_release -cs + OUTPUT_VARIABLE RELEASE_CODENAME + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if (NOT RELEASE_CODENAME STREQUAL "trusty") + set(GTEST_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") + endif() + endif() endif() set(JEMALLOC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") set(LZ4_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") @@ -653,13 +666,19 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) set(GTEST_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) + set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GMOCK_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GMOCK_MAIN_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_STATIC_LIBRARY_SUFFIX}") + if (MSVC AND NOT ARROW_USE_STATIC_CRT) set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON) endif() ExternalProject_Add(googletest_ep URL ${GTEST_SOURCE_URL} - BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB} + BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB} ${GMOCK_STATIC_LIB} ${GMOCK_MAIN_STATIC_LIB} CMAKE_ARGS ${GTEST_CMAKE_ARGS} ${EP_LOG_OPTIONS}) else() @@ -668,28 +687,50 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) endif() message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}") + message(STATUS "GMock include dir: ${GMOCK_INCLUDE_DIR}") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) + # Conflicts in header files seem to either cause apple to have + # a bad boost symbol, or trusty to use CPP_TOOLCHAIN's header + # file for gmock (and the vendored version is 1.8.0 and conda is + # 1.8.1) + if (APPLE) + include_directories(SYSTEM ${GMOCK_INCLUDE_DIR}) + else() + include_directories(BEFORE SYSTEM ${GMOCK_INCLUDE_DIR}) + endif() if(GTEST_STATIC_LIB) message(STATUS "GTest static library: ${GTEST_STATIC_LIB}") + message(STATUS "GMock static library: ${GMOCK_STATIC_LIB}") ADD_THIRDPARTY_LIB(gtest STATIC_LIB ${GTEST_STATIC_LIB}) ADD_THIRDPARTY_LIB(gtest_main STATIC_LIB ${GTEST_MAIN_STATIC_LIB}) + ADD_THIRDPARTY_LIB(gmock + STATIC_LIB ${GMOCK_STATIC_LIB}) + ADD_THIRDPARTY_LIB(gmock_main + STATIC_LIB ${GMOCK_MAIN_STATIC_LIB}) set(GTEST_LIBRARY gtest_static) set(GTEST_MAIN_LIBRARY gtest_main_static) + set(GMOCK_LIBRARY gmock_static) + set(GMOCK_MAIN_LIBRARY gmock_main_static) else() message(STATUS "GTest shared library: ${GTEST_SHARED_LIB}") + message(STATUS "GMock shared library: ${GMOCK_SHARED_LIB}") ADD_THIRDPARTY_LIB(gtest SHARED_LIB ${GTEST_SHARED_LIB}) ADD_THIRDPARTY_LIB(gtest_main SHARED_LIB ${GTEST_MAIN_SHARED_LIB}) set(GTEST_LIBRARY gtest_shared) set(GTEST_MAIN_LIBRARY gtest_main_shared) + set(GMOCK_LIBRARY gmock_shared) + set(GMOCK_MAIN_LIBRARY gmock_main_shared) endif() if(GTEST_VENDORED) add_dependencies(${GTEST_LIBRARY} googletest_ep) add_dependencies(${GTEST_MAIN_LIBRARY} googletest_ep) + add_dependencies(${GMOCK_LIBRARY} googletest_ep) + add_dependencies(${GMOCK_MAIN_LIBRARY} googletest_ep) endif() endif() diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index 8129441b41fa1..f850a296976a5 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -71,15 +71,8 @@ TEST(TestDatum, ImplicitConstructors) { class TestInvokeBinaryKernel : public ComputeFixture, public TestBase {}; -class DummyBinaryKernel : public BinaryKernel { - Status Call(FunctionContext* ctx, const Datum& left, const Datum& right, - Datum* out) override { - return Status::OK(); - } -}; - TEST_F(TestInvokeBinaryKernel, Exceptions) { - DummyBinaryKernel kernel; + MockBinaryKernel kernel; std::vector outputs; std::shared_ptr table; vector values1 = {true, false, true}; diff --git a/cpp/src/arrow/compute/test-util.h b/cpp/src/arrow/compute/test-util.h index e2bda698a9bff..b406a710b45bc 100644 --- a/cpp/src/arrow/compute/test-util.h +++ b/cpp/src/arrow/compute/test-util.h @@ -21,11 +21,14 @@ #include #include +#include + #include "arrow/array.h" #include "arrow/memory_pool.h" #include "arrow/type.h" #include "arrow/compute/context.h" +#include "arrow/compute/kernel.h" namespace arrow { namespace compute { @@ -38,6 +41,16 @@ class ComputeFixture { FunctionContext ctx_; }; +class MockUnaryKernel : public UnaryKernel { + public: + MOCK_METHOD3(Call, Status(FunctionContext* ctx, const Datum& input, Datum* out)); +}; + +class MockBinaryKernel : public BinaryKernel { + MOCK_METHOD4(Call, Status(FunctionContext* ctx, const Datum& left, const Datum& right, + Datum* out)); +}; + template std::shared_ptr _MakeArray(const std::shared_ptr& type, const std::vector& values, diff --git a/python/manylinux1/scripts/build_gtest.sh b/python/manylinux1/scripts/build_gtest.sh index 5b29f5ee535c8..723b59bddb7b6 100755 --- a/python/manylinux1/scripts/build_gtest.sh +++ b/python/manylinux1/scripts/build_gtest.sh @@ -25,13 +25,13 @@ pushd googletest-release-${GTEST_VERSION} mkdir build_so pushd build_so -cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON -DBUILD_SHARED_LIBS=ON -DBUILD_GMOCK=OFF -GNinja -DCMAKE_INSTALL_PREFIX=/usr .. +cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON -DBUILD_SHARED_LIBS=ON -DBUILD_GMOCK=ON -GNinja -DCMAKE_INSTALL_PREFIX=/usr .. ninja install popd mkdir build_a pushd build_a -cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_GMOCK=OFF -GNinja -DCMAKE_INSTALL_PREFIX=/usr .. +cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_GMOCK=ON -GNinja -DCMAKE_INSTALL_PREFIX=/usr .. ninja install popd From fb113225387be68e268d585c7216d21871a7ed62 Mon Sep 17 00:00:00 2001 From: Yosuke Shiro Date: Sat, 2 Feb 2019 05:51:11 +0900 Subject: [PATCH 19/20] ARROW-4235: [GLib] Use "column_builder" in GArrowRecordBatchBuilder Author: Yosuke Shiro Closes #3540 from shiro615/glib-use-column-builder and squashes the following commits: a1741ccd Use column_builders in GArrowRecordBatchBuilderPrivate 23021ae7 Add garrow_record_batch_builder_get_n_columns() fc388b7f Use garrow_record_batch_builder_get_column_builder() in garrow_record_batch_builder_get_field() 03b681d8 Add test case for column_builders 60928755 Make public column_builders 1680c663 Add garrow_record_batch_builder_get_column_builder() --- c_glib/arrow-glib/table-builder.cpp | 56 ++++++++++++++++--- c_glib/arrow-glib/table-builder.h | 13 +++++ c_glib/test/test-record-batch-builder.rb | 18 +++--- .../lib/arrow/record-batch-builder.rb | 17 +++--- .../test/test-record-batch-builder.rb | 9 +++ 5 files changed, 86 insertions(+), 27 deletions(-) diff --git a/c_glib/arrow-glib/table-builder.cpp b/c_glib/arrow-glib/table-builder.cpp index e87314bf52b9f..5e004a55d8a05 100644 --- a/c_glib/arrow-glib/table-builder.cpp +++ b/c_glib/arrow-glib/table-builder.cpp @@ -41,7 +41,7 @@ G_BEGIN_DECLS typedef struct GArrowRecordBatchBuilderPrivate_ { arrow::RecordBatchBuilder *record_batch_builder; - GPtrArray *fields; + GPtrArray *column_builders; } GArrowRecordBatchBuilderPrivate; enum { @@ -63,13 +63,13 @@ garrow_record_batch_builder_constructed(GObject *object) { auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(object); auto arrow_builder = priv->record_batch_builder; - auto n_fields = arrow_builder->num_fields(); - priv->fields = g_ptr_array_new_full(n_fields, g_object_unref); - for (int i = 0; i < n_fields; ++i) { + auto n_columns = arrow_builder->num_fields(); + priv->column_builders = g_ptr_array_new_full(n_columns, g_object_unref); + for (int i = 0; i < n_columns; ++i) { auto arrow_array_builder = arrow_builder->GetField(i); auto array_builder = garrow_array_builder_new_raw(arrow_array_builder); garrow_array_builder_release_ownership(array_builder); - g_ptr_array_add(priv->fields, array_builder); + g_ptr_array_add(priv->column_builders, array_builder); } G_OBJECT_CLASS(garrow_record_batch_builder_parent_class)->constructed(object); @@ -80,7 +80,7 @@ garrow_record_batch_builder_finalize(GObject *object) { auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(object); - g_ptr_array_free(priv->fields, TRUE); + g_ptr_array_free(priv->column_builders, TRUE); delete priv->record_batch_builder; G_OBJECT_CLASS(garrow_record_batch_builder_parent_class)->finalize(object); @@ -223,9 +223,26 @@ garrow_record_batch_builder_get_schema(GArrowRecordBatchBuilder *builder) * Returns: The number of fields. * * Since: 0.8.0 + * + * Deprecated: 0.13.0: + * Use garrow_record_batch_builder_get_n_columns() instead. */ gint garrow_record_batch_builder_get_n_fields(GArrowRecordBatchBuilder *builder) +{ + return garrow_record_batch_builder_get_n_columns(builder); +} + +/** + * garrow_record_batch_builder_get_n_columns: + * @builder: A #GArrowRecordBatchBuilder. + * + * Returns: The number of columns. + * + * Since: 0.13.0 + */ +gint +garrow_record_batch_builder_get_n_columns(GArrowRecordBatchBuilder *builder) { auto arrow_builder = garrow_record_batch_builder_get_raw(builder); return arrow_builder->num_fields(); @@ -241,23 +258,44 @@ garrow_record_batch_builder_get_n_fields(GArrowRecordBatchBuilder *builder) * the `i`-th field on success, %NULL on out of index. * * Since: 0.8.0 + * + * Deprecated: 0.13.0: + * Use garrow_record_batch_builder_get_column_builder() instead. */ GArrowArrayBuilder * garrow_record_batch_builder_get_field(GArrowRecordBatchBuilder *builder, gint i) +{ + return garrow_record_batch_builder_get_column_builder(builder, i); +} + +/** + * garrow_record_batch_builder_get_column_builder: + * @builder: A #GArrowRecordBatchBuilder. + * @i: The column index. If it's negative, index is counted backward + * from the end of the columns. `-1` means the last column. + * + * Returns: (transfer none) (nullable): The #GArrowArrayBuilder for + * the `i`-th column on success, %NULL on out of index. + * + * Since: 0.13.0 + */ +GArrowArrayBuilder * +garrow_record_batch_builder_get_column_builder(GArrowRecordBatchBuilder *builder, + gint i) { auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(builder); if (i < 0) { - i += priv->fields->len; + i += priv->column_builders->len; } if (i < 0) { return NULL; } - if (static_cast(i) >= priv->fields->len) { + if (static_cast(i) >= priv->column_builders->len) { return NULL; } - return GARROW_ARRAY_BUILDER(g_ptr_array_index(priv->fields, i)); + return GARROW_ARRAY_BUILDER(g_ptr_array_index(priv->column_builders, i)); } /** diff --git a/c_glib/arrow-glib/table-builder.h b/c_glib/arrow-glib/table-builder.h index d05525e54f52e..a76793953c55d 100644 --- a/c_glib/arrow-glib/table-builder.h +++ b/c_glib/arrow-glib/table-builder.h @@ -45,9 +45,22 @@ void garrow_record_batch_builder_set_initial_capacity(GArrowRecordBatchBuilder * gint64 capacity); GArrowSchema *garrow_record_batch_builder_get_schema(GArrowRecordBatchBuilder *builder); +#ifndef GARROW_DISABLE_DEPRECATED +GARROW_DEPRECATED_IN_0_13_FOR(garrow_record_batch_builder_get_n_columns) gint garrow_record_batch_builder_get_n_fields(GArrowRecordBatchBuilder *builder); +#endif +GARROW_AVAILABLE_IN_0_13 +gint +garrow_record_batch_builder_get_n_columns(GArrowRecordBatchBuilder *builder); +#ifndef GARROW_DISABLE_DEPRECATED +GARROW_DEPRECATED_IN_0_13_FOR(garrow_record_batch_builder_get_column_builder) GArrowArrayBuilder *garrow_record_batch_builder_get_field(GArrowRecordBatchBuilder *builder, gint i); +#endif +GARROW_AVAILABLE_IN_0_13 +GArrowArrayBuilder * +garrow_record_batch_builder_get_column_builder(GArrowRecordBatchBuilder *builder, + gint i); GArrowRecordBatch *garrow_record_batch_builder_flush(GArrowRecordBatchBuilder *builder, GError **error); diff --git a/c_glib/test/test-record-batch-builder.rb b/c_glib/test/test-record-batch-builder.rb index 030cc78a01555..ce8efdffd98d8 100644 --- a/c_glib/test/test-record-batch-builder.rb +++ b/c_glib/test/test-record-batch-builder.rb @@ -37,27 +37,27 @@ def test_schema assert_equal(@schema, @builder.schema) end - def test_n_fields - assert_equal(@fields.size, @builder.n_fields) + def test_n_columns + assert_equal(@fields.size, @builder.n_columns) end - sub_test_case("#get_field") do + sub_test_case("#get_column_builder") do def test_valid assert_equal(Arrow::BooleanArrayBuilder, - @builder.get_field(0).class) + @builder.get_column_builder(0).class) end def test_negative assert_equal(Arrow::Int32ArrayBuilder, - @builder.get_field(-1).class) + @builder.get_column_builder(-1).class) end def test_too_negative - assert_nil(@builder.get_field(-@fields.size - 1)) + assert_nil(@builder.get_column_builder(-@fields.size - 1)) end def test_too_large - assert_nil(@builder.get_field(@fields.size)) + assert_nil(@builder.get_column_builder(@fields.size)) end end @@ -68,7 +68,7 @@ def test_flush "point" => build_int32_array([1, -1, 0]), } arrays.each_with_index do |(_, array), i| - @builder.get_field(i).append_values(array.values, []) + @builder.get_column_builder(i).append_values(array.values, []) end assert_equal(build_record_batch(arrays), @builder.flush) @@ -78,7 +78,7 @@ def test_flush "point" => build_int32_array([10, -10]), } arrays.each_with_index do |(_, array), i| - @builder.get_field(i).append_values(array.values, []) + @builder.get_column_builder(i).append_values(array.values, []) end assert_equal(build_record_batch(arrays), @builder.flush) diff --git a/ruby/red-arrow/lib/arrow/record-batch-builder.rb b/ruby/red-arrow/lib/arrow/record-batch-builder.rb index dba16b3b8116d..dc20312f203f6 100644 --- a/ruby/red-arrow/lib/arrow/record-batch-builder.rb +++ b/ruby/red-arrow/lib/arrow/record-batch-builder.rb @@ -65,7 +65,7 @@ def append(*values) # @since 0.12.0 def append_records(records) - n = n_fields + n = n_columns columns = n.times.collect do [] end @@ -99,17 +99,16 @@ def append_columns(columns) end end + # @since 0.13.0 + def column_builders + @column_builders ||= n_columns.times.collect do |i| + get_column_builder(i) + end + end + private def resolve_name(name) @name_to_index[name.to_s] end - - # TODO: Make public with good name. Is column_builders good enough? - # builders? sub_builders? - def column_builders - @column_builders ||= n_fields.times.collect do |i| - get_field(i) - end - end end end diff --git a/ruby/red-arrow/test/test-record-batch-builder.rb b/ruby/red-arrow/test/test-record-batch-builder.rb index 7cd1f8cee7a16..988e0204345a4 100644 --- a/ruby/red-arrow/test/test-record-batch-builder.rb +++ b/ruby/red-arrow/test/test-record-batch-builder.rb @@ -112,5 +112,14 @@ def setup arrays), @builder.flush) end + + test("#column_builders") do + column_builders = [ + @builder.get_column_builder(0), + @builder.get_column_builder(1), + ] + assert_equal(column_builders, + @builder.column_builders) + end end end From f08e109ff009e911bb772e5b0490c7cacf02140e Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Fri, 1 Feb 2019 16:12:12 -0800 Subject: [PATCH 20/20] ARROW-4455: [Plasma] Suppress class-memaccess warnings cpp/src/plasma/store.cc: In member function 'arrow::Status plasma::PlasmaStore::ProcessMessage(plasma::Client*)': cpp/src/plasma/store.cc:767:36: error: 'void* memset(void*, int, size_t)' clearing an object of type 'struct plasma::PlasmaObject' with no trivial copy-assignment; use assignment or value-initialization instead [-Werror=class-memaccess] memset(&object, 0, sizeof(object)); ^ In file included from cpp/src/plasma/eviction_policy.h:27, from cpp/src/plasma/store.h:30, from cpp/src/plasma/store.cc:29: cpp/src/plasma/plasma.h:75:8: note: 'struct plasma::PlasmaObject' declared here struct PlasmaObject { ^~~~~~~~~~~~ cpp/src/plasma/test/serialization_tests.cc: In function 'plasma::PlasmaObject plasma::random_plasma_object()': cpp/src/plasma/test/serialization_tests.cc:68:36: error: 'void* memset(void*, int, size_t)' clearing an object of type 'struct plasma::PlasmaObject' with no trivial copy-assignment; use assignment or value-initialization instead [-Werror=class-memaccess] memset(&object, 0, sizeof(object)); ^ In file included from cpp/src/plasma/test/serialization_tests.cc:25: cpp/src/plasma/plasma.h:75:8: note: 'struct plasma::PlasmaObject' declared here struct PlasmaObject { ^~~~~~~~~~~~ cpp/src/plasma/test/serialization_tests.cc: In member function 'virtual void plasma::PlasmaSerialization_CreateReply_Test::TestBody()': cpp/src/plasma/test/serialization_tests.cc:110:38: error: 'void* memset(void*, int, size_t)' clearing an object of type 'struct plasma::PlasmaObject' with no trivial copy-assignment; use assignment or value-initialization instead [-Werror=class-memaccess] memset(&object2, 0, sizeof(object2)); ^ In file included from cpp/src/plasma/test/serialization_tests.cc:25: cpp/src/plasma/plasma.h:75:8: note: 'struct plasma::PlasmaObject' declared here struct PlasmaObject { ^~~~~~~~~~~~ Author: Kouhei Sutou Closes #3543 from kou/plasma-suppress-class-memaccess-warning and squashes the following commits: 34ce66c0c Suppress class-memaccess warnings --- cpp/src/plasma/store.cc | 4 +--- cpp/src/plasma/test/serialization_tests.cc | 6 ++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc index f84e89077ca69..745e336049e8b 100644 --- a/cpp/src/plasma/store.cc +++ b/cpp/src/plasma/store.cc @@ -762,9 +762,7 @@ Status PlasmaStore::ProcessMessage(Client* client) { uint8_t* input = input_buffer_.data(); size_t input_size = input_buffer_.size(); ObjectID object_id; - PlasmaObject object; - // TODO(pcm): Get rid of the following. - memset(&object, 0, sizeof(object)); + PlasmaObject object = {}; // Process the different types of requests. switch (type) { diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc index 66d651d2923bf..4fb3f9a5ed376 100644 --- a/cpp/src/plasma/test/serialization_tests.cc +++ b/cpp/src/plasma/test/serialization_tests.cc @@ -64,8 +64,7 @@ std::vector read_message_from_file(int fd, MessageType message_type) { PlasmaObject random_plasma_object(void) { unsigned int seed = static_cast(time(NULL)); int random = rand_r(&seed); - PlasmaObject object; - memset(&object, 0, sizeof(object)); + PlasmaObject object = {}; object.store_fd = random + 7; object.data_offset = random + 1; object.metadata_offset = random + 2; @@ -106,8 +105,7 @@ TEST(PlasmaSerialization, CreateReply) { ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, PlasmaError::OK, mmap_size1)); std::vector data = read_message_from_file(fd, MessageType::PlasmaCreateReply); ObjectID object_id2; - PlasmaObject object2; - memset(&object2, 0, sizeof(object2)); + PlasmaObject object2 = {}; int store_fd; int64_t mmap_size2; ARROW_CHECK_OK(ReadCreateReply(data.data(), data.size(), &object_id2, &object2,