8000 Merge branch 'main' into feature/4.0 · dotnet/machinelearning@e72e985 · GitHub
[go: up one dir, main page]
Skip to content

Commit e72e985

Browse files
JakeRadMSFTJakeRadMSFT
committed
Merge branch 'main' into feature/4.0
2 parents 9aea1ce + 64d7ebd commit e72e985

File tree

187 files changed

+109842
-25333
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

187 files changed

+109842
-25333
lines changed

‎Microsoft.ML.sln

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Tokenizers.Tes
168168
EndProject
169169
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Microsoft.ML.FSharp.Tests", "test\Microsoft.ML.FSharp.Tests\Microsoft.ML.FSharp.Tests.fsproj", "{041CB5CD-5832-413E-A894-D9DBED210B16}"
170170
EndProject
171+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis.PerformanceTests", "test\Microsoft.Data.Analysis.PerformanceTests\Microsoft.Data.Analysis.PerformanceTests.csproj", "{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}"
172+
EndProject
171173
Global
172174
GlobalSection(SolutionConfigurationPlatforms) = preSolution
173175
Debug|Any CPU = Debug|Any CPU
@@ -788,6 +790,14 @@ Global
788790
{041CB5CD-5832-413E-A894-D9DBED210B16}.Release|Any CPU.Build.0 = Release|Any CPU
789791
{041CB5CD-5832-413E-A894-D9DBED210B16}.Release|x64.ActiveCfg = Release|Any CPU
790792
{041CB5CD-5832-413E-A894-D9DBED210B16}.Release|x64.Build.0 = Release|Any CPU
793+
{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
794+
{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Debug|Any CPU.Build.0 = Debug|Any CPU
795+
{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Debug|x64.ActiveCfg = Debug|Any CPU
796+
{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Debug|x64.Build.0 = Debug|Any CPU
797+
{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Release|Any CPU.ActiveCfg = Release|Any CPU
798+
{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Release|Any CPU.Build.0 = Release|Any CPU
799+
{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Release|x64.ActiveCfg = Release|Any CPU
800+
{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD}.Release|x64.Build.0 = Release|Any CPU
791801
EndGlobalSection
792802
GlobalSection(SolutionProperties) = preSolution
793803
HideSolutionNode = FALSE
@@ -870,6 +880,7 @@ Global
870880
{BBC3A950-BD68-45AC-9DBD-A8F4D8847745} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
871881
{C3D82402-F207-4F19-8C57-5AF0FBAF9682} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
872882
{041CB5CD-5832-413E-A894-D9DBED210B16} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
883+
{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
873884
EndGlobalSection
874885
GlobalSection(ExtensibilityGlobals) = postSolution
875886
SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

‎THIRD-PARTY-NOTICES.TXT

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,3 +85,24 @@ distributed under the License is distributed on an "AS IS" BASIS,
8585
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
8686
See the License for the specific language governing permissions and
8787
limitations under the License.
88+
89+
90+
License notice for BitUtility
91+
------------------------------------------
92+
93+
https://github.com/apache/arrow/blob/main/csharp/src/Apache.Arrow/BitUtility.cs
94+
95+
Licensed to the Apache Software Foundation (ASF) under one or more
96+
contributor license agreements. See the NOTICE file distributed with
97+
this work for additional information regarding copyright ownership.
98+
The ASF licenses this file to You under the Apache License, Version 2.0
99+
(the "License"); you may not use this file except in compliance with
100+
the License. You may obtain a copy of the License at
101+
102+
http://www.apache.org/licenses/LICENSE-2.0
103+
104+
Unless required by applicable law or agreed to in writing, software
105+
distributed under the License is distributed on an "AS IS" BASIS,
106+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
107+
See the License for the specific language governing permissions and
108+
limitations under the License.

‎build/ci/job-template.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ jobs:
4444
${{ if eq(parameters.codeCoverage, 'false') }}:
4545
hardLink: '/p:CreateHardLinksForCopyLocalIfPossible=True'
4646
testTargetFramework: '/p:TestTargetFramework=$(_targetFramework)'
47-
CODECOV_TOKEN: 03031e35-fe75-4e4c-87ee-e919ae601748
4847
strategy:
4948
matrix:
5049
${{ if eq(parameters.customMatrixes, '') }}:
@@ -68,7 +67,7 @@ jobs:
6867
steps:
6968
# Extra MacOS step required to install OS-specific dependencies
7069
- ${{ if and(contains(parameters.pool.vmImage, 'macOS'), not(contains(parameters.name, 'cross'))) }}:
71-
- script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=TRUE && brew update && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
70+
- script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=TRUE && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
7271
displayName: Install MacOS build dependencies
7372
# Extra Apple MacOS step required to install OS-specific dependencies
7473
- ${{ if and(contains(parameters.pool.vmImage, 'macOS'), contains(parameters.name, 'cross')) }}:

‎build/libomp.rb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ class Libomp < Formula
1515
end
1616

1717
depends_on "cmake" => :build
18-
depends_on macos: :yosemite
1918

2019
def install
2120
system "cmake", ".", *std_cmake_args

‎build/vsts-ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ stages:
100100
pool:
101101
vmImage: macOS-12
102102
steps:
103-
- script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 && brew update && rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
103+
- script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 && rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
104104
displayName: Install build dependencies
105105
# Only build native assets to avoid conflicts.
106106
- script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true

‎eng/Versions.props

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@
1616
<MicrosoftBclAsyncInterfacesVersion>6.0.0</MicrosoftBclAsyncInterfacesVersion>
1717
<MicrosoftExtensionsVersion>2.1.0</MicrosoftExtensionsVersion>
1818
<MicrosoftExtensionsDependencyInjectionVersion>6.0.0</MicrosoftExtensionsDependencyInjectionVersion>
19-
<SkiaSharpVersion>2.88.3</SkiaSharpVersion>
19+
<NuGetVersion>6.7.0</NuGetVersion>
20+
<SkiaSharpVersion>2.88.6</SkiaSharpVersion>
2021
<SystemBuffersVersion>4.5.1</SystemBuffersVersion>
2122
<SystemCodeDomVersion>4.5.0</SystemCodeDomVersion>
2223
<SystemCollectionsImmutableVersion>1.5.0</SystemCollectionsImmutableVersion>
24+
<SystemConfigurationConfigurationManagerVersion>6.0.1</SystemConfigurationConfigurationManagerVersion>
2325
<SystemIOFileSystemAccessControl>4.5.0</SystemIOFileSystemAccessControl>
2426
<SystemMemoryVersion>4.5.5</SystemMemoryVersion>
2527
<SystemReflectionEmitLightweightVersion>4.3.0</SystemReflectionEmitLightweightVersion>
@@ -30,13 +32,13 @@
3032
<SystemTextJsonVersion>6.0.1</SystemTextJsonVersion>
3133
<SystemThreadingChannelsVersion>4.7.1</SystemThreadingChannelsVersion>
3234
<!-- Other product dependencies -->
33-
<ApacheArrowVersion>2.0.0</ApacheArrowVersion>
35+
<ApacheArrowVersion>11.0.0</ApacheArrowVersion>
3436
<GoogleProtobufVersion>3.19.6</GoogleProtobufVersion>
3537
<LightGBMVersion>2.3.1</LightGBMVersion>
3638
<MicrosoftCodeAnalysisAnalyzersVersion>3.3.0</MicrosoftCodeAnalysisAnalyzersVersion>
3739
<MicrosoftCodeAnalysisCSharpVersion>3.9.0</MicrosoftCodeAnalysisCSharpVersion>
38-
<MicrosoftDotNetInteractiveFormattingVersion>1.0.0-beta.22504.6</MicrosoftDotNetInteractiveFormattingVersion>
39-
<MicrosoftDotNetInteractiveVersion>1.0.0-beta.22504.6</MicrosoftDotNetInteractiveVersion>
40+
<MicrosoftDotNetInteractiveFormattingVersion>1.0.0-beta.23461.4</MicrosoftDotNetInteractiveFormattingVersion>
41+
<MicrosoftDotNetInteractiveVersion>1.0.0-beta.23461.4</MicrosoftDotNetInteractiveVersion>
4042
<MicrosoftMLOnnxRuntimeVersion>1.14.0</MicrosoftMLOnnxRuntimeVersion>
4143
<MlNetMklDepsVersion>0.0.0.12</MlNetMklDepsVersion>
4244
<!--
@@ -74,11 +76,11 @@
7476
<SystemCompositionVersion>1.2.0</SystemCompositionVersion>
7577
<!-- Test-only Dependencies -->
7678
<ApprovalTestsVersion>5.4.7</ApprovalTestsVersion>
77-
<BenchmarkDotNetVersion>0.12.0</BenchmarkDotNetVersion>
79+
<BenchmarkDotNetVersion>0.13.1</BenchmarkDotNetVersion>
7880
<DotNetRuntime60Version>6.0.9</DotNetRuntime60Version>
7981
<DotNetRuntime80Version>8.0.0-preview.3.23174.8</DotNetRuntime80Version>
8082
<FluentAssertionVersion>5.10.2</FluentAssertionVersion>
81-
<MicrosoftCodeAnalysisTestingVersion>1.1.2-beta1.22512.1</MicrosoftCodeAnalysisTestingVersion>
83+
<MicrosoftCodeAnalysisTestingVersion>1.1.2-beta1.23431.1</MicrosoftCodeAnalysisTestingVersion>
8284
<MicrosoftDotNetXUnitExtensionsVersion>8.0.0-beta.23265.1</MicrosoftDotNetXUnitExtensionsVersion>
8385
<MicrosoftExtensionsDependencyModelVersion>2.1.0</MicrosoftExtensionsDependencyModelVersion>
8486
<MicrosoftExtensionsTestVersion>3.0.1</MicrosoftExtensionsTestVersion>
@@ -87,7 +89,7 @@
8789
<MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
8890
<MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
8991
<SystemDataSqlClientVersion>4.6.1</SystemDataSqlClientVersion>
90-
<SystemDataSQLiteCoreVersion>1.0.113</SystemDataSQLiteCoreVersion>
92+
<SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
9193
<XunitCombinatorialVersion>1.2.7</XunitCombinatorialVersion>
9294
<XUnitVersion>2.4.2</XUnitVersion>
9395
<!-- Opt-out repo features -->

‎src/Microsoft.Data.Analysis.Interactive/TabularDataResourceExtensions.cs

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6+
using System.Collections.Generic;
67
using System.Linq;
78
using Microsoft.Data.Analysis;
89

@@ -22,19 +23,32 @@ public static DataFrame ToDataFrame(this TabularDataResource tabularDataResource
2223

2324
foreach (var fieldDescriptor in tabularDataResource.Schema.Fields)
2425
{
26+
var fieldName = fieldDescriptor.Name;
27+
var column = tabularDataResource.Data.Select(row =>
28+
{
29+
if (row is IDictionary<string, object> dictionary)
30+
{
31+
return dictionary[fieldName];
32+
}
33+
else
34+
{
35+
return row.FirstOrDefault(kvp => kvp.Key == fieldName).Value;
36+
}
37+
});
38+
2539
switch (fieldDescriptor.Type)
2640
{
2741
case TableSchemaFieldType.Number:
28-
dataFrame.Columns.Add(new DoubleDataFrameColumn(fieldDescriptor.Name, tabularDataResource.Data.Select(d => Convert.ToDouble(d[fieldDescriptor.Name]))));
42+
dataFrame.Columns.Add(new DoubleDataFrameColumn(fieldDescriptor.Name, column.Select(Convert.ToDouble)));
2943
break;
3044
case TableSchemaFieldType.Integer:
31-
dataFrame.Columns.Add(new Int64DataFrameColumn(fieldDescriptor.Name, tabularDataResource.Data.Select(d => Convert.ToInt64(d[fieldDescriptor.Name]))));
45+
dataFrame.Columns.Add(new Int64DataFrameColumn(fieldDescriptor.Name, column.Select(Convert.ToInt64)));
3246
break;
3347
case TableSchemaFieldType.Boolean:
34-
dataFrame.Columns.Add(new BooleanDataFrameColumn(fieldDescriptor.Name, tabularDataResource.Data.Select(d => Convert.ToBoolean(d[fieldDescriptor.Name]))));
48+
dataFrame.Columns.Add(new BooleanDataFrameColumn(fieldDescriptor.Name, column.Select(Convert.ToBoolean)));
3549
break;
3650
case TableSchemaFieldType.String:
37-
dataFrame.Columns.Add(new StringDataFrameColumn(fieldDescriptor.Name, tabularDataResource.Data.Select(d => Convert.ToString(d[fieldDescriptor.Name]))));
51+
dataFrame.Columns.Add(new StringDataFrameColumn(fieldDescriptor.Name, column.Select(Convert.ToString)));
3852
break;
3953
default:
4054
throw new ArgumentOutOfRangeException();

‎src/Microsoft.Data.Analysis/ArrayUtility.cs

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.Collections.Generic;
7+
using System.Text;
8+
9+
namespace Microsoft.Data.Analysis
10+
{
11+
internal static class ArrayUtility
12+
{
13+
// Maximum size of one-dimensional array.
14+
// See: https://msdn.microsoft.com/en-us/library/hh285054(v=vs.110).aspx
15+
// Polyfilling Array.MaxLength API for netstandard2.0
16+
public const int ArrayMaxSize = 0X7FEFFFFF;
17+
}
18+
}

‎src/Microsoft.Data.Analysis/ArrowStringDataFrameColumn.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -213,9 +213,9 @@ private void Append(ReadOnlySpan<byte> value)
213213
_offsetsBuffers.Add(mutableOffsetsBuffer);
214214
mutableOffsetsBuffer.Append(0);
215215
}
216-
mutableDataBuffer.EnsureCapacity(value.Length);
217-
value.CopyTo(mutableDataBuffer.RawSpan.Slice(mutableDataBuffer.Length));
218-
mutableDataBuffer.Length += value.Length;
216+
var startIndex = mutableDataBuffer.Length;
217+
mutableDataBuffer.IncreaseSize(value.Length);
218+
value.CopyTo(mutableDataBuffer.RawSpan.Slice(startIndex));
219219
mutableOffsetsBuffer.Append(mutableOffsetsBuffer[mutableOffsetsBuffer.Length - 1] + value.Length);
220220
}
221221
SetValidityBit(Length - 1, value != default);

‎src/Microsoft.Data.Analysis/BitUtility.cs

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.Diagnostics;
7+
using System.Runtime.CompilerServices;
8+
using System.Runtime.InteropServices;
9+
10+
namespace Microsoft.Data.Analysis
11+
{
12+
// License for BitUtility
13+
// --------------------------------------
14+
// This class is based on the code from Apache Arrow project
15+
// https://github.com/apache/arrow/blob/main/csharp/src/Apache.Arrow/BitUtility.cs
16+
// that is available in the public domain inder Apache-2.0 license.
17+
// You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0
18+
internal static class BitUtility
19+
{
20+
private static ReadOnlySpan<byte> PopcountTable => new byte[] {
21+
0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
22+
1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
23+
1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
24+
2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
25+
1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
26+
2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
27+
2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
28+
3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8,
29+
};
30+
31+
private static ReadOnlySpan<byte> BitMask => new byte[] {
32+
1, 2, 4, 8, 16, 32, 64, 128
33+
};
34+
35+
// Faster to use when we already have a span since it avoids indexing
36+
public static bool IsValid(ReadOnlySpan<byte> bitMapBufferSpan, int index)
37+
{
38+
var nullBitMapSpanIndex = index / 8;
39+
var thisBitMap = bitMapBufferSpan[nullBitMapSpanIndex];
40+
return IsBitSet(thisBitMap, index);
41+
}
42+
43+
public static bool IsBitSet(byte curBitMap, int index)
44+
{
45+
return ((curBitMap >> (index & 7)) & 1) != 0;
46+
}
47+
48+
public static bool IsBitClear(byte curBitMap, int index)
49+
{
50+
return ((curBitMap >> (index & 7)) & 1) == 0;
51+
}
52+
53+
public static bool GetBit(byte data, int index) =>
54+
((data >> index) & 1) != 0;
55+
56+
public static bool GetBit(ReadOnlySpan<byte> data, int index) =>
57+
(data[index / 8] & BitMask[index % 8]) != 0;
58+
59+
public static void ClearBit(Span<byte> data, int index)
60+
{
61+
data[index / 8] &= (byte)~BitMask[index % 8];
62+
}
63+
64+
public static void SetBit(Span<byte> data, int index)
65+
{
66+
data[index / 8] |= BitMask[index % 8];
67+
}
68+
69+
public static void SetBit(Span<byte> data, long index, bool value)
70+
{
71+
int idx = (int)(index / 8);
72+
int mod = (int)(index % 8);
73+
data[idx] = value
74+
? (byte)(data[idx] | 10000 BitMask[mod])
75+
: (byte)(data[idx] & ~BitMask[mod]);
76+
}
77+
78+
/// <summary>
79+
/// Set the number of bits in a span of bytes starting
80+
/// at a specific index, and limiting to length.
81+
/// </summary>
82+
/// <param name="data">Span to set bits value.</param>
83+
/// <param name="index">Bit index to start counting from.</param>
84+
/// <param name="length">Maximum of bits in the span to consider.</param>
85+
/// <param name="value">Bit value.</param>
86+
public static void SetBits(Span<byte> data, long index, long length, bool value)
87+
{
88+
if (length == 0)
89+
return;
90+
91+
var endBitIndex = index + length - 1;
92+
93+
// Use simpler method if there aren't many values
94+
if (length < 20)
95+
{
96+
for (var i = index; i <= endBitIndex; i++)
97+
{
98+
SetBit(data, i, value);
99+
}
100+
return;
101+
}
102+
103+
// Otherwise do the work to figure out how to copy whole bytes
104+
var startByteIndex = (int)(index / 8);
105+
var startBitOffset = (int)(index % 8);
106+
var endByteIndex = (int)(endBitIndex / 8);
107+
var endBitOffset = (int)(endBitIndex % 8);
108+
109+
// If the starting index and ending index are not byte-aligned,
110+
// we'll need to set bits the slow way. If they are
111+
// byte-aligned, and for all other bytes in the 'middle', we
112+
// can use a faster byte-aligned set.
113+
var fullByteStartIndex = startBitOffset == 0 ? startByteIndex : startByteIndex + 1;
114+
var fullByteEndIndex = endBitOffset == 7 ? endByteIndex : endByteIndex - 1;
115+
116+
// Bits we will be using to finish up the first byte
117+
if (startBitOffset != 0)
118+
{
119+
var slice = data.Slice(startByteIndex, 1);
120+
for (var i = startBitOffset; i <= 7; i++)
121+
SetBit(slice, i, value);
122+
}
123+
124+
if (fullByteEndIndex >= fullByteStartIndex)
125+
{
126+
var slice = data.Slice(fullByteStartIndex, fullByteEndIndex - fullByteStartIndex + 1);
127+
byte fill = (byte)(value ? 0xFF : 0x00);
128+
129+
slice.Fill(fill);
130+
}
131+
132+
if (endBitOffset != 7)
133+
{
134+
var slice = data.Slice(endByteIndex, 1);
135+
for (int i = 0; i <= endBitOffset; i++)
136+
SetBit(slice, i, value);
137+
}
138+
}
139+
140+
public static void ElementwiseAnd(ReadOnlySpan<byte> left, ReadOnlySpan<byte> right, Span<byte> result)
141+
{
142+
for (var i = 0; i < left.Length; i++)
143+
result[i] = (byte)(left[i] & right[i]);
144+
}
145+
146+
/// <summary>
147+
/// Returns the population count (number of bits set) in a span of bytes starting
148+
/// at 0 bit and limiting to length of bits.
149+
/// </summary>
150+
/// <param name="span"></param>
151+
/// <param name="length"></param>
152+
/// <returns></returns>
153+
public static long GetBitCount(ReadOnlySpan<byte> span, long length)
154+
{
155+
var endByteIndex = (int)(length / 8);
156+
157+
Debug.Assert(span.Length > endByteIndex);
158+
159+
long count = 0;
160+
for (var i = 0; i < endByteIndex; i++)
161+
count += PopcountTable[span[i]];
162+
163+
var endBitOffset = (int)(length % 8);
164+
165+
if (endBitOffset != 0)
166+
{
167+
var partialByte = span[endByteIndex];
168+
for (var j = 0; j < endBitOffset; j++)
169+
{
170+
count += GetBit(partialByte, j) ? 0 : 1;
171+
}
172+
}
173+
174+
return count;
175+
}
176+
}
177+
}

0 commit comments

Comments
 (0)
0