Update Core to 2.21 (#397)

* Regenerate bindings for 2.21.

* Expose WKB and WKT data types.

* Support specifying custom GitHub organizations in the MSBuild scripts.

This will be useful for test releases on forks.

* Simplify specifying the Core version to use.

* Update tests for ls recursive

* Bump versions

---------

Co-authored-by: Shaun Reed <shaunrd0@gmail.com>

Author: teo-tsirpanis

.github/workflows/nightly.yml

@@ -27,7 +27,7 @@ jobs:
           - os: windows-latest
             platform: windows-x86_64
           - tag: dev
-        tag: [release-2.20, dev]
+        tag: [release-2.21, dev]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout TileDB
@@ -72,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        tag: [release-2.20, dev]
+        tag: [release-2.21, dev]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout TileDB-CSharp
@@ -113,7 +113,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        tag: [release-2.20, dev]
+        tag: [release-2.21, dev]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout TileDB-CSharp

Directory.Packages.props

@@ -2,7 +2,9 @@
   <PropertyGroup>
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
     <TileDBNativePackageName>TileDB.Native</TileDBNativePackageName>
-    <TileDBNativePackageVersion>[2.20.0,2.21.0)</TileDBNativePackageVersion>
+    <TileDBNativeVersionMajor>2</TileDBNativeVersionMajor>
+    <TileDBNativeVersionMinor>21</TileDBNativeVersionMinor>
+    <TileDBNativePackageVersion>[$(TileDBNativeVersionMajor).$(TileDBNativeVersionMinor).0,$(TileDBNativeVersionMajor).$([MSBuild]::Add($(TileDBNativeVersionMinor), 1)).0)</TileDBNativePackageVersion>
 
     <!-- The DevelopmentBuild property switches to the locally built native packages.
     They have a different name to avoid publishing them by mistake, and to

scripts/generate-bindings/GenerateBindings.proj

@@ -9,6 +9,7 @@
     <InputDir>$(TempDir)/download/include</InputDir>
     <RspFile>$(TempDir)/generate.rsp</RspFile>
     <AccessSpecifier>%2A=internal</AccessSpecifier>
+    <GitHubOrganization>TileDB-Inc</GitHubOrganization>
   </PropertyGroup>
 
   <ItemGroup>
@@ -69,7 +70,7 @@
       Text="Version and VersionTag must be specified." />
 
     <PropertyGroup>
-      <DownloadUrl>https://github.com/TileDB-Inc/TileDB/releases/download/$(Version)/TileDB-windows-x86_64-$(Version)-$(VersionTag).zip</DownloadUrl>
+      <DownloadUrl>https://github.com/$(GitHubOrganization)/TileDB/releases/download/$(Version)/TileDB-windows-x86_64-$(Version)-$(VersionTag).zip</DownloadUrl>
       <ExtractDir>$(TempDir)/download</ExtractDir>
     </PropertyGroup>
     <DownloadFile

scripts/nuget/GenerateNuGetPackages.proj

@@ -5,6 +5,7 @@
     <RuntimeJson>$(TempDir)/runtime.json</RuntimeJson>
     <MetapackageProject>./TileDB.Native.proj</MetapackageProject>
     <NativePackageTemplateProject>./TileDB.Native.runtime.template.proj</NativePackageTemplateProject>
+    <GitHubOrganization>TileDB-Inc</GitHubOrganization>
   </PropertyGroup>
 
   <PropertyGroup Condition="$(DevelopmentBuild) == true">
@@ -47,7 +48,7 @@
       Text="Version and VersionTag must be specified." />
 
     <PropertyGroup>
-      <DownloadUrl>https://github.com/TileDB-Inc/TileDB/releases/download/$(Version)/TileDB-%(NativePlatform.Identity)-$(Version)-$(VersionTag).%(NativePlatform.ArchiveExtension)</DownloadUrl>
+      <DownloadUrl>https://github.com/$(GitHubOrganization)/TileDB/releases/download/$(Version)/TileDB-%(NativePlatform.Identity)-$(Version)-$(VersionTag).%(NativePlatform.ArchiveExtension)</DownloadUrl>
       <ExtractDir>$(TempDir)/%(NativePlatform.Identity)</ExtractDir>
     </PropertyGroup>
     <DownloadFile

sources/TileDB.CSharp/Enums.cs

@@ -502,7 +502,23 @@ public enum DataType : uint
     /// On generic methods operating to objects of this datatype,
     /// this datatype can be used with <see cref="byte"/> or <see cref="bool"/>.
     /// </remarks>
-    Boolean = tiledb_datatype_t.TILEDB_BOOL
+    Boolean = tiledb_datatype_t.TILEDB_BOOL,
+    /// <summary>
+    /// A geometry in Well-Known Binary (WKB) format.
+    /// </summary>
+    /// <remarks>
+    /// On generic methods operating to objects of this datatype,
+    /// this datatype can be used with <see cref="byte"/>.
+    /// </remarks>
+    GeometryWkb = tiledb_datatype_t.TILEDB_GEOM_WKB,
+    /// <summary>
+    /// A geometry in Well-Known Text (WKT) format.
+    /// </summary>
+    /// <remarks>
+    /// On generic methods operating to objects of this datatype,
+    /// this datatype can be used with <see cref="byte"/>.
+    /// </remarks>
+    GeometryWkt = tiledb_datatype_t.TILEDB_GEOM_WKT
 }
 
 /// <summary>
@@ -1315,6 +1331,9 @@ internal static Type DataTypeToNumericType(DataType datatype)
                 return typeof(byte);
             case DataType.Boolean:
                 return typeof(byte);
+            case DataType.GeometryWkb:
+            case DataType.GeometryWkt:
+                return typeof(byte);
             default:
                 return typeof(byte);
         }

sources/TileDB.CSharp/Interop/tiledb_datatype_t.cs

@@ -52,5 +52,7 @@ public enum tiledb_datatype_t
         TILEDB_TIME_AS = 39,
         TILEDB_BLOB = 40,
         TILEDB_BOOL = 41,
+        TILEDB_GEOM_WKB = 42,
+        TILEDB_GEOM_WKT = 43,
     }
 }

sources/TileDB.CSharp/TileDB.CSharp.csproj

@@ -6,12 +6,12 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Nullable>enable</Nullable>
     <RootNamespace>TileDB.CSharp</RootNamespace>
-    <Version>5.10.0</Version>
+    <Version>5.11.0</Version>
     <Description>C# wrapper of the TileDB Embedded universal data engine.</Description>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <EnablePackageValidation>true</EnablePackageValidation>
-    <PackageValidationBaselineVersion>5.9.0</PackageValidationBaselineVersion>
+    <PackageValidationBaselineVersion>5.10.0</PackageValidationBaselineVersion>
     <NoWarn>$(NoWarn);TILEDB0012;TILEDB0013;TILEDB0014</NoWarn>
   </PropertyGroup>
 

sources/TileDB.CSharp/VFS.cs

@@ -439,7 +439,7 @@ public void VisitChildren<T>(string uri, Func<string, T, bool> callback, T callb
     /// <param name="callbackArg">An argument that will be passed to <paramref name="callback"/>.</param>
     /// <typeparam name="T">The type of <paramref name="callbackArg"/>.</typeparam>
     /// <remarks>
-    /// This operation is supported only on URIs to AWS S3.
+    /// This operation is supported only on local filesystem and AWS S3 URIs.
     /// </remarks>
     public void VisitChildrenRecursive<T>(string uri, Func<string, ulong, T, bool> callback, T callbackArg)
     {

tests/TileDB.CSharp.Test/VFSTest.cs

@@ -157,23 +157,23 @@ public void TestVisitPropagatesExceptions()
     [TestMethod]
     public void TestVisitRecursive()
     {
-        Assert.Inconclusive("VisitChildrenRecursive is currently supported only on S3.");
-
         using var dir = new TemporaryDirectory("vfs-visit");
 
         using var vfs = new VFS();
+        vfs.CreateDir(Path.Combine(dir, "dir1"));
         vfs.Touch(Path.Combine(dir, "dir1/file1"));
+        vfs.CreateDir(Path.Combine(dir, "dir2"));
         vfs.Touch(Path.Combine(dir, "dir2/file2"));
-        vfs.Touch(Path.Combine(dir, "dir3/file3"));
+        vfs.CreateDir(Path.Combine(dir, "dir3"));
+        System.IO.File.WriteAllBytes(Path.Combine(dir, "dir3/file3"), "12345"u8.ToArray());
 
         int i = 0;
 
         vfs.VisitChildrenRecursive(dir, (uri, size, arg) =>
         {
             string path = new Uri(uri).LocalPath;
-            Assert.IsTrue(System.IO.File.Exists(path));
-            Assert.AreEqual((string)dir, Path.GetDirectoryName(path));
-            Assert.AreEqual(0ul, size);
+            Assert.IsTrue(Directory.Exists(path) || System.IO.File.Exists(path));
+            Assert.AreEqual(path.EndsWith("file3") ? 5ul : 0ul, size);
             Assert.AreEqual(555, arg);
 
             i++;
@@ -186,16 +186,13 @@ public void TestVisitRecursive()
     [TestMethod]
     public void TestVisitRecursivePropagatesExceptions()
     {
-        Assert.Inconclusive("VisitChildrenRecursive is currently supported only on S3.");
-
         using var dir = new TemporaryDirectory("vfs-visit-exceptions");
 
         const string ExceptionKey = "foo";
 
         using var vfs = new VFS();
+        vfs.CreateDir(Path.Combine(dir, "dir1"));
         vfs.Touch(Path.Combine(dir, "dir1/file1"));
-        vfs.Touch(Path.Combine(dir, "dir2/file2"));
-        vfs.Touch(Path.Combine(dir, "dir3/file3"));
 
         int i = 0;