dotnet · JakeRadMSFT · Oct 5, 2023 · Jun 28, 2023 · Jun 27, 2023 · Jul 6, 2023
diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml
@@ -2,6 +2,20 @@
 # ML.NET's PR validation build
 ################################################################################
 
+pr:
+  branches:
+    include:
+    - main
+    - feature/*
+    - release/*
+
+trigger:
+  branches:
+    include:
+    - main
+    - feature/*
+    - release/*
+
 resources:
   containers:
   - container: CentosContainer

diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln
@@ -168,6 +168,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Tokenizers.Tes
 EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Microsoft.ML.FSharp.Tests", "test\Microsoft.ML.FSharp.Tests\Microsoft.ML.FSharp.Tests.fsproj", "{041CB5CD-5832-413E-A894-D9DBED210B16}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis.PerformanceTests", "test\Microsoft.Data.Analysis.PerformanceTests\Microsoft.Data.Analysis.PerformanceTests.csproj", "{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -788,6 +790,14 @@ Global
 		{041CB5CD-5832-413E-A894-D9DBED210B16}.Release|Any CPU.Build.0 = Release|Any CPU
 		{041CB5CD-5832-413E-A894-D9DBED210B16}.Release|x64.ActiveCfg = Release|Any CPU
 		{041CB5CD-5832-413E-A894-D9DBED210B16}.Release|x64.Build.0 = Release|Any CPU
+		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Debug|x64.Build.0 = Debug|Any CPU
+		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Release|x64.ActiveCfg = Release|Any CPU
+		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -870,6 +880,7 @@ Global
 		{BBC3A950-BD68-45AC-9DBD-A8F4D8847745} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{C3D82402-F207-4F19-8C57-5AF0FBAF9682} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{041CB5CD-5832-413E-A894-D9DBED210B16} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT
@@ -85,3 +85,24 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+
+License notice for BitUtility
+------------------------------------------
+
+https://github.com/apache/arrow/blob/main/csharp/src/Apache.Arrow/BitUtility.cs
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/build/.night-build.yml b/build/.night-build.yml
@@ -15,9 +15,8 @@ schedules:
   branches:
     include:
     - main
-    - releases/1.6.0
-    - features/automl
-    - features/integrationPackage
+    - feature/*
+    - release/*
   always: true
 
 resources:

diff --git a/build/.outer-loop-build.yml b/build/.outer-loop-build.yml
@@ -15,9 +15,8 @@ schedules:
   branches:
     include:
     - main
-    - releases/1.6.0
-    - features/automl
-    - features/integrationPackage
+    - feature/*
+    - release/*
   always: true
 
 

diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml
@@ -44,7 +44,6 @@ jobs:
       ${{ if eq(parameters.codeCoverage, 'false') }}:
         hardLink: '/p:CreateHardLinksForCopyLocalIfPossible=True'
       testTargetFramework: '/p:TestTargetFramework=$(_targetFramework)'
-      CODECOV_TOKEN: 03031e35-fe75-4e4c-87ee-e919ae601748
     strategy:
       matrix:
         ${{ if eq(parameters.customMatrixes, '') }}:
@@ -68,7 +67,7 @@ jobs:
     steps:
     # Extra MacOS step required to install OS-specific dependencies
     - ${{ if and(contains(parameters.pool.vmImage, 'macOS'), not(contains(parameters.name, 'cross')))  }}:
-      - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=TRUE && brew update && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
+      - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=TRUE && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
         displayName: Install MacOS build dependencies
     # Extra Apple MacOS step required to install OS-specific dependencies
     - ${{ if and(contains(parameters.pool.vmImage, 'macOS'), contains(parameters.name, 'cross'))  }}:

diff --git a/build/codecoverage-ci.yml b/build/codecoverage-ci.yml
@@ -2,6 +2,20 @@
 # ML.NET's Code Coverage validation build
 ################################################################################
 
+pr:
+  branches:
+    include:
+    - main
+    - feature/*
+    - release/*
+
+trigger:
+  branches:
+    include:
+    - main
+    - feature/*
+    - release/*
+
 jobs:
 - template: /build/ci/job-template.yml
   parameters:

diff --git a/build/libomp.rb b/build/libomp.rb
@@ -15,7 +15,6 @@ class Libomp < Formula
   end
 
   depends_on "cmake" => :build
-  depends_on macos: :yosemite
 
   def install
     system "cmake", ".", *std_cmake_args

diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml
@@ -100,7 +100,7 @@ stages:
     pool:
       vmImage: macOS-12
     steps:
-    - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 && brew update &&  rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
+    - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 &&  rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
       displayName: Install build dependencies
     # Only build native assets to avoid conflicts.
     - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true

diff --git a/eng/Versions.props b/eng/Versions.props
@@ -9,17 +9,19 @@
         https://github.com/dotnet/arcade/blob/c788ffa83b088cafe9dbffc1cbc8155ba88b2553/Documentation/CorePackages/Versioning.md#output
     -->
     <DotNetUseShippingVersions>true</DotNetUseShippingVersions>
-    <VersionPrefix>3.0.0</VersionPrefix>
+    <VersionPrefix>4.0.0</VersionPrefix>
     <PreReleaseVersionLabel>preview</PreReleaseVersionLabel>
     <AssemblyVersion>1.0.0.0</AssemblyVersion>
     <!-- .NET Runtime product dependencies -->
     <MicrosoftBclAsyncInterfacesVersion>6.0.0</MicrosoftBclAsyncInterfacesVersion>
     <MicrosoftExtensionsVersion>2.1.0</MicrosoftExtensionsVersion>
     <MicrosoftExtensionsDependencyInjectionVersion>6.0.0</MicrosoftExtensionsDependencyInjectionVersion>
-    <SkiaSharpVersion>2.88.3</SkiaSharpVersion>
+    <NuGetVersion>6.7.0</NuGetVersion>
+    <SkiaSharpVersion>2.88.6</SkiaSharpVersion>
     <SystemBuffersVersion>4.5.1</SystemBuffersVersion>
     <SystemCodeDomVersion>4.5.0</SystemCodeDomVersion>
     <SystemCollectionsImmutableVersion>1.5.0</SystemCollectionsImmutableVersion>
+    <SystemConfigurationConfigurationManagerVersion>6.0.1</SystemConfigurationConfigurationManagerVersion>
     <SystemIOFileSystemAccessControl>4.5.0</SystemIOFileSystemAccessControl>
     <SystemMemoryVersion>4.5.5</SystemMemoryVersion>
     <SystemReflectionEmitLightweightVersion>4.3.0</SystemReflectionEmitLightweightVersion>
@@ -30,13 +32,13 @@
     <SystemTextJsonVersion>6.0.1</SystemTextJsonVersion>
     <SystemThreadingChannelsVersion>4.7.1</SystemThreadingChannelsVersion>
     <!-- Other product dependencies -->
-    <ApacheArrowVersion>2.0.0</ApacheArrowVersion>
+    <ApacheArrowVersion>11.0.0</ApacheArrowVersion>
     <GoogleProtobufVersion>3.19.6</GoogleProtobufVersion>
     <LightGBMVersion>2.3.1</LightGBMVersion>
     <MicrosoftCodeAnalysisAnalyzersVersion>3.3.0</MicrosoftCodeAnalysisAnalyzersVersion>
     <MicrosoftCodeAnalysisCSharpVersion>3.9.0</MicrosoftCodeAnalysisCSharpVersion>
-    <MicrosoftDotNetInteractiveFormattingVersion>1.0.0-beta.22504.6</MicrosoftDotNetInteractiveFormattingVersion>
-    <MicrosoftDotNetInteractiveVersion>1.0.0-beta.22504.6</MicrosoftDotNetInteractiveVersion>
+    <MicrosoftDotNetInteractiveFormattingVersion>1.0.0-beta.23461.4</MicrosoftDotNetInteractiveFormattingVersion>
+    <MicrosoftDotNetInteractiveVersion>1.0.0-beta.23461.4</MicrosoftDotNetInteractiveVersion>
     <MicrosoftMLOnnxRuntimeVersion>1.14.0</MicrosoftMLOnnxRuntimeVersion>
     <MlNetMklDepsVersion>0.0.0.12</MlNetMklDepsVersion>
     <!--  
@@ -74,11 +76,11 @@
     <SystemCompositionVersion>1.2.0</SystemCompositionVersion>
     <!-- Test-only Dependencies -->
     <ApprovalTestsVersion>5.4.7</ApprovalTestsVersion>
-    <BenchmarkDotNetVersion>0.12.0</BenchmarkDotNetVersion>
+    <BenchmarkDotNetVersion>0.13.1</BenchmarkDotNetVersion>
     <DotNetRuntime60Version>6.0.9</DotNetRuntime60Version>
     <DotNetRuntime80Version>8.0.0-preview.3.23174.8</DotNetRuntime80Version>
     <FluentAssertionVersion>5.10.2</FluentAssertionVersion>
-    <MicrosoftCodeAnalysisTestingVersion>1.1.2-beta1.22512.1</MicrosoftCodeAnalysisTestingVersion>
+    <MicrosoftCodeAnalysisTestingVersion>1.1.2-beta1.23431.1</MicrosoftCodeAnalysisTestingVersion>
     <MicrosoftDotNetXUnitExtensionsVersion>8.0.0-beta.23265.1</MicrosoftDotNetXUnitExtensionsVersion>
     <MicrosoftExtensionsDependencyModelVersion>2.1.0</MicrosoftExtensionsDependencyModelVersion>
     <MicrosoftExtensionsTestVersion>3.0.1</MicrosoftExtensionsTestVersion>
@@ -87,7 +89,7 @@
     <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
     <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
     <SystemDataSqlClientVersion>4.6.1</SystemDataSqlClientVersion>
-    <SystemDataSQLiteCoreVersion>1.0.113</SystemDataSQLiteCoreVersion>
+    <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
     <XunitCombinatorialVersion>1.2.7</XunitCombinatorialVersion>
     <XUnitVersion>2.4.2</XUnitVersion>
     <!-- Opt-out repo features -->

diff --git a/src/Microsoft.Data.Analysis.Interactive/TabularDataResourceExtensions.cs b/src/Microsoft.Data.Analysis.Interactive/TabularDataResourceExtensions.cs
@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using Microsoft.Data.Analysis;
 
@@ -22,19 +23,32 @@ public static DataFrame ToDataFrame(this TabularDataResource tabularDataResource
 
             foreach (var fieldDescriptor in tabularDataResource.Schema.Fields)
             {
+                var fieldName = fieldDescriptor.Name;
+                var column = tabularDataResource.Data.Select(row =>
+                {
+                    if (row is IDictionary<string, object> dictionary)
+                    {
+                        return dictionary[fieldName];
+                    }
+                    else
+                    {
+                        return row.FirstOrDefault(kvp => kvp.Key == fieldName).Value;
+                    }
+                });
+
                 switch (fieldDescriptor.Type)
                 {
                     case TableSchemaFieldType.Number:
-                        dataFrame.Columns.Add(new DoubleDataFrameColumn(fieldDescriptor.Name, tabularDataResource.Data.Select(d => Convert.ToDouble(d[fieldDescriptor.Name]))));
+                        dataFrame.Columns.Add(new DoubleDataFrameColumn(fieldDescriptor.Name, column.Select(Convert.ToDouble)));
                         break;
                     case TableSchemaFieldType.Integer:
-                        dataFrame.Columns.Add(new Int64DataFrameColumn(fieldDescriptor.Name, tabularDataResource.Data.Select(d => Convert.ToInt64(d[fieldDescriptor.Name]))));
+                        dataFrame.Columns.Add(new Int64DataFrameColumn(fieldDescriptor.Name, column.Select(Convert.ToInt64)));
                         break;
                     case TableSchemaFieldType.Boolean:
-                        dataFrame.Columns.Add(new BooleanDataFrameColumn(fieldDescriptor.Name, tabularDataResource.Data.Select(d => Convert.ToBoolean(d[fieldDescriptor.Name]))));
+                        dataFrame.Columns.Add(new BooleanDataFrameColumn(fieldDescriptor.Name, column.Select(Convert.ToBoolean)));
                         break;
                     case TableSchemaFieldType.String:
-                        dataFrame.Columns.Add(new StringDataFrameColumn(fieldDescriptor.Name, tabularDataResource.Data.Select(d => Convert.ToString(d[fieldDescriptor.Name]))));
+                        dataFrame.Columns.Add(new StringDataFrameColumn(fieldDescriptor.Name, column.Select(Convert.ToString)));
                         break;
                     default:
                         throw new ArgumentOutOfRangeException();

diff --git a/src/Microsoft.Data.Analysis/ArrayUtility.cs b/src/Microsoft.Data.Analysis/ArrayUtility.cs
@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.Data.Analysis
+{
+    internal static class ArrayUtility
+    {
+        // Maximum size of one-dimensional array.
+        // See: https://msdn.microsoft.com/en-us/library/hh285054(v=vs.110).aspx
+        // Polyfilling Array.MaxLength API for netstandard2.0
+        public const int ArrayMaxSize = 0X7FEFFFFF;
+    }
+}
diff --git a/src/Microsoft.Data.Analysis/ArrowStringDataFrameColumn.cs b/src/Microsoft.Data.Analysis/ArrowStringDataFrameColumn.cs
@@ -213,9 +213,9 @@ private void Append(ReadOnlySpan<byte> value)
                     _offsetsBuffers.Add(mutableOffsetsBuffer);
                     mutableOffsetsBuffer.Append(0);
                 }
-                mutableDataBuffer.EnsureCapacity(value.Length);
-                value.CopyTo(mutableDataBuffer.RawSpan.Slice(mutableDataBuffer.Length));
-                mutableDataBuffer.Length += value.Length;
+                var startIndex = mutableDataBuffer.Length;
+                mutableDataBuffer.IncreaseSize(value.Length);
+                value.CopyTo(mutableDataBuffer.RawSpan.Slice(startIndex));
                 mutableOffsetsBuffer.Append(mutableOffsetsBuffer[mutableOffsetsBuffer.Length - 1] + value.Length);
             }
             SetValidityBit(Length - 1, value != default);