ARROW-13168: [C++][R] Enable runtime timezone database for Windows

.github/workflows/cpp.yml

@@ -239,6 +239,9 @@ jobs:
         with:
           fetch-depth: 0
           submodules: recursive
+      - name: Download Timezone Database
+        shell: bash
+        run: ci/scripts/download_tz_database.sh
       - name: Build
         shell: bash
         run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
@@ -319,6 +322,9 @@ jobs:
         run: |
           export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS
           ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
+      - name: Download Timezone Database
+        shell: bash
+        run: ci/scripts/download_tz_database.sh
       - name: Download MinIO
         shell: msys2 {0}
         run: |

ci/appveyor-cpp-setup.bat

@@ -115,3 +115,17 @@ powershell.exe -Command "Start-Process clcache-server" || exit /B
 if "%ARROW_S3%" == "ON" (
     appveyor DownloadFile https://dl.min.io/server/minio/release/windows-amd64/minio.exe -FileName C:\Windows\Minio.exe || exit /B
 )
+
+
+@rem
+@rem Download IANA Timezone Database for unit tests
+@rem
+@rem (Doc section: Download timezone database)
+curl https://data.iana.org/time-zones/releases/tzdata2021e.tar.gz --output tzdata.tar.gz
+mkdir tzdata
+tar --extract --file tzdata.tar.gz --directory tzdata
+move tzdata %USERPROFILE%\Downloads\tzdata
+@rem Also need Windows timezone mapping
+curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml ^
+  --output %USERPROFILE%\Downloads\tzdata\windowsZones.xml
+@rem (Doc section: Download timezone database)

ci/scripts/download_tz_database.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+# Download database
+curl https://data.iana.org/time-zones/releases/tzdata2021e.tar.gz --output ~/Downloads/tzdata2021e.tar.gz
+
+# Extract
+mkdir -p ~/Downloads/tzdata
+tar --extract --file ~/Downloads/tzdata2021e.tar.gz --directory ~/Downloads/tzdata
+
+# Download Windows timezone mapping
+curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml --output ~/Downloads/tzdata/windowsZones.xml

cpp/src/arrow/compute/kernels/scalar_cast_string.cc

@@ -150,12 +150,6 @@ struct TemporalToStringCastFunctor<O, TimestampType> {
             return Status::OK();
           }));
     } else {
-#ifdef _WIN32
-      // TODO(ARROW-13168):
-      return Status::NotImplemented(
-          "Casting a timestamp with time zone to string is not yet supported on "
-          "Windows.");
-#else
       switch (ty.unit()) {
         case TimeUnit::SECOND:
           RETURN_NOT_OK(ConvertZoned<std::chrono::seconds>(input, timezone, &builder));
@@ -176,7 +170,6 @@ struct TemporalToStringCastFunctor<O, TimestampType> {
           DCHECK(false);
           return Status::NotImplemented("Unimplemented time unit");
       }
-#endif
     }
     std::shared_ptr<Array> output_array;
     RETURN_NOT_OK(builder.Finish(&output_array));

cpp/src/arrow/compute/kernels/scalar_cast_test.cc

@@ -34,6 +34,7 @@
 #include "arrow/testing/extension_type.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
+#include "arrow/testing/util.h"
 #include "arrow/type.h"
 #include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
@@ -1146,6 +1147,16 @@ constexpr char kTimestampSecondsJson[] =
 constexpr char kTimestampExtremeJson[] =
     R"(["1677-09-20T00:00:59.123456", "2262-04-13T23:23:23.999999"])";
 
+class CastTimezone : public ::testing::Test {
+ protected:
+  void SetUp() override {
+#ifdef _WIN32
+    // Initialize timezone database on Windows
+    ASSERT_OK(InitTestTimezoneDatabase());
+#endif
+  }
+};
+
 TEST(Cast, TimestampToDate) {
   // See scalar_temporal_test.cc
   auto timestamps = ArrayFromJSON(timestamp(TimeUnit::NANO), kTimestampJson);
@@ -1181,12 +1192,7 @@ TEST(Cast, TimestampToDate) {
   }
 }
 
-TEST(Cast, ZonedTimestampToDate) {
-#ifdef _WIN32
-  // TODO(ARROW-13168): we lack tzdb on Windows
-  GTEST_SKIP() << "ARROW-13168: no access to timezone database on Windows";
-#endif
-
+TEST_F(CastTimezone, ZonedTimestampToDate) {
   {
     // See TestZoned in scalar_temporal_test.cc
     auto timestamps =
@@ -1377,12 +1383,7 @@ TEST(Cast, TimestampToTime) {
   }
 }
 
-TEST(Cast, ZonedTimestampToTime) {
-#ifdef _WIN32
-  // TODO(ARROW-13168): we lack tzdb on Windows
-  GTEST_SKIP() << "ARROW-13168: no access to timezone database on Windows";
-#endif
-
+TEST_F(CastTimezone, ZonedTimestampToTime) {
   CheckCast(ArrayFromJSON(timestamp(TimeUnit::NANO, "Pacific/Marquesas"), kTimestampJson),
             ArrayFromJSON(time64(TimeUnit::NANO), R"([
           52259123456789, 50003999999999, 56480001001001, 65000000000000,
@@ -1573,8 +1574,7 @@ TEST(Cast, TimestampToString) {
   }
 }
 
-#ifndef _WIN32
-TEST(Cast, TimestampWithZoneToString) {
+TEST_F(CastTimezone, TimestampWithZoneToString) {
   for (auto string_type : {utf8(), large_utf8()}) {
     CheckCast(
         ArrayFromJSON(timestamp(TimeUnit::SECOND, "UTC"), "[-30610224000, -5364662400]"),
@@ -1608,21 +1608,6 @@ TEST(Cast, TimestampWithZoneToString) {
             R"(["1968-11-30 13:30:44.123456789-0700", "2016-02-29 10:42:23.456789246-0700"])"));
   }
 }
-#else
-// TODO(ARROW-13168): we lack tzdb on Windows
-TEST(Cast, TimestampWithZoneToString) {
-  for (auto string_type : {utf8(), large_utf8()}) {
-    ASSERT_RAISES(NotImplemented, Cast(ArrayFromJSON(timestamp(TimeUnit::SECOND, "UTC"),
-                                                     "[-34226955, 1456767743]"),
-                                       CastOptions::Safe(string_type)));
-
-    ASSERT_RAISES(NotImplemented,
-                  Cast(ArrayFromJSON(timestamp(TimeUnit::SECOND, "America/Phoenix"),
-                                     "[-34226955, 1456767743]"),
-                       CastOptions::Safe(string_type)));
-  }
-}
-#endif
 
 TEST(Cast, DateToDate) {
   auto day_32 = ArrayFromJSON(date32(), "[0, null, 100, 1, 10]");

cpp/src/arrow/compute/kernels/scalar_temporal_test.cc

@@ -24,6 +24,7 @@
 #include "arrow/compute/kernels/test_util.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/matchers.h"
+#include "arrow/testing/util.h"
 #include "arrow/type.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/formatting.h"
@@ -407,6 +408,14 @@ class ScalarTemporalTest : public ::testing::Test {
   RoundTemporalOptions round_to_15_quarters =
       RoundTemporalOptions(15, CalendarUnit::QUARTER);
   RoundTemporalOptions round_to_15_years = RoundTemporalOptions(15, CalendarUnit::YEAR);
+
+ protected:
+  void SetUp() override {
+#ifdef _WIN32
+    // Initialize timezone database on Windows
+    ASSERT_OK(InitTestTimezoneDatabase());
+#endif
+  }
 };
 
 TEST_F(ScalarTemporalTest, TestTemporalComponentExtractionAllTemporalTypes) {
@@ -564,8 +573,6 @@ TEST_F(ScalarTemporalTest, TestOutsideNanosecondRange) {
   CheckScalarUnary("subsecond", unit, times, float64(), subsecond);
 }
 
-#ifndef _WIN32
-// TODO: We should test on windows once ARROW-13168 is resolved.
 TEST_F(ScalarTemporalTest, TestIsLeapYear) {
   auto is_leap_year_marquesas =
       "[false, true, false, false, false, false, false, false, false, false, false, "
@@ -792,7 +799,6 @@ TEST_F(ScalarTemporalTest, TestNonexistentTimezone) {
     ASSERT_RAISES(Invalid, Subsecond(timestamp_array));
   }
 }
-#endif
 
 TEST_F(ScalarTemporalTest, Week) {
   auto unit = timestamp(TimeUnit::NANO);
@@ -1607,8 +1613,6 @@ TEST_F(ScalarTemporalTest, TestTemporalDifferenceErrors) {
       CallFunction("weeks_between", {arr1, arr1}, &options));
 }
 
-// TODO: We should test on windows once ARROW-13168 is resolved.
-#ifndef _WIN32
 TEST_F(ScalarTemporalTest, TestAssumeTimezone) {
   std::string timezone_utc = "UTC";
   std::string timezone_kolkata = "Asia/Kolkata";
@@ -1875,6 +1879,9 @@ TEST_F(ScalarTemporalTest, StrftimeCLocale) {
 }
 
 TEST_F(ScalarTemporalTest, StrftimeOtherLocale) {
+#ifdef _WIN32
+  GTEST_SKIP() << "There is a known bug in strftime for locales on Windows (ARROW-15922)";
+#else
   if (!LocaleExists("fr_FR.UTF-8")) {
     GTEST_SKIP() << "locale 'fr_FR.UTF-8' doesn't exist on this system";
   }
@@ -1886,6 +1893,7 @@ TEST_F(ScalarTemporalTest, StrftimeOtherLocale) {
       ["01 janvier 1970 00:00:59,123", "18 août 2021 15:11:50,456", null])";
   CheckScalarUnary("strftime", timestamp(TimeUnit::MILLI, "UTC"), milliseconds, utf8(),
                    expected, &options);
+#endif
 }
 
 TEST_F(ScalarTemporalTest, StrftimeInvalidLocale) {
@@ -2579,7 +2587,6 @@ TEST_F(ScalarTemporalTest, TestCeilFloorRoundTemporalKolkata) {
   CheckScalarUnary("round_temporal", unit, times, unit, round_1_hours, &round_to_1_hours);
   CheckScalarUnary("round_temporal", unit, times, unit, round_2_hours, &round_to_2_hours);
 }
-#endif  // !_WIN32
 
 }  // namespace compute
 }  // namespace arrow

cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc

@@ -1046,7 +1046,6 @@ struct RoundTemporal {
 // ----------------------------------------------------------------------
 // Convert timestamps to a string representation with an arbitrary format
 
-#ifndef _WIN32
 Result<std::locale> GetLocale(const std::string& locale) {
   try {
     return std::locale(locale.c_str());
@@ -1130,18 +1129,6 @@ struct Strftime {
     return Status::OK();
   }
 };
-#else
-// TODO(ARROW-13168)
-template <typename Duration, typename InType>
-struct Strftime {
-  static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
-    return Status::NotImplemented("Strftime not yet implemented on windows.");
-  }
-  static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
-    return Status::NotImplemented("Strftime not yet implemented on windows.");
-  }
-};
-#endif
 
 // ----------------------------------------------------------------------
 // Convert timestamps from local timestamp without a timezone to timestamps with a

cpp/src/arrow/config.cc

@@ -21,6 +21,7 @@
 
 #include "arrow/util/config.h"
 #include "arrow/util/cpu_info.h"
+#include "arrow/vendored/datetime.h"
 
 namespace arrow {
 
@@ -62,6 +63,8 @@ std::string MakeSimdLevelString(QueryFlagFunction&& query_flag) {
   }
 }
 
+util::optional<std::string> timezone_db_path;
+
 };  // namespace
 
 const BuildInfo& GetBuildInfo() { return kBuildInfo; }
@@ -73,7 +76,32 @@ RuntimeInfo GetRuntimeInfo() {
       MakeSimdLevelString([&](int64_t flags) { return cpu_info->IsSupported(flags); });
   info.detected_simd_level =
       MakeSimdLevelString([&](int64_t flags) { return cpu_info->IsDetected(flags); });
+  info.using_os_timezone_db = USE_OS_TZDB;
+#if !USE_OS_TZDB
+  info.timezone_db_path = timezone_db_path;
+#else
+  info.timezone_db_path = util::optional<std::string>();
+#endif
   return info;
 }
 
+Status Initialize(const GlobalOptions& options) noexcept {
+  if (options.timezone_db_path.has_value()) {
+#if !USE_OS_TZDB
+    try {
+      arrow_vendored::date::set_install(options.timezone_db_path.value());
+      arrow_vendored::date::reload_tzdb();
+    } catch (const std::runtime_error& e) {
+      return Status::IOError(e.what());
+    }
+    timezone_db_path = options.timezone_db_path.value();
+#else
+    return Status::Invalid(
+        "Arrow was set to use OS timezone database at compile time, "
+        "so a downloaded database cannot be provided at runtime.");
+#endif  // !USE_OS_TZDB
+  }
+  return Status::OK();
+}
+
 }  // namespace arrow

cpp/src/arrow/config.h

@@ -19,7 +19,9 @@
 
 #include <string>
 
+#include "arrow/status.h"
 #include "arrow/util/config.h"  // IWYU pragma: export
+#include "arrow/util/optional.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
@@ -62,6 +64,13 @@ struct RuntimeInfo {
 
   /// The SIMD level available on the OS and CPU
   std::string detected_simd_level;
+
+  /// Whether using the OS-based timezone database
+  /// This is set at compile-time.
+  bool using_os_timezone_db;
+
+  /// The path to the timezone database; by default None.
+  util::optional<std::string> timezone_db_path;
 };
 
 /// \brief Get runtime build info.
@@ -77,4 +86,13 @@ const BuildInfo& GetBuildInfo();
 ARROW_EXPORT
 RuntimeInfo GetRuntimeInfo();
 
+struct GlobalOptions {
+  /// Path to text timezone database. This is only configurable on Windows,
+  /// which does not have a compatible OS timezone database.
+  util::optional<std::string> timezone_db_path;
+};
+
+ARROW_EXPORT
+Status Initialize(const GlobalOptions& options) noexcept;
+
 }  // namespace arrow