10000 [WIP] Broadcast serialization bug by Niharikadutta · Pull Request #561 · dotnet/spark · GitHub
[go: up one dir, main page]
Skip to content

[WIP] Broadcast serialization bug #561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

[WIP] Broadcast serialization bug #561

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
03b7939
Adding section for UDF serialization
Niharikadutta Apr 20, 2020 10000
4ef693d
removing guides from master
Niharikadutta Apr 20, 2020
81145ca
Merge latest from master
Niharikadutta May 6, 2020
e4b81af
merging latest from master
Niharikadutta May 7, 2020
4c32173
Merge remote-tracking branch 'upstream/master'
Niharikadutta Jun 2, 2020
4987a09
Merge remote-tracking branch 'upstream/master'
Niharikadutta Jun 14, 2020
ca9612e
Merge remote-tracking branch 'upstream/master'
Niharikadutta Jun 16, 2020
f581c86
Merge remote-tracking branch 'upstream/master'
Niharikadutta Jun 20, 2020
f564495
Marking classes as serializable to fix the Broadcast serialization issue
Niharikadutta Jun 22, 2020
b761998
Marking necessary classes as serializable to make broadcast work in n…
Niharikadutta Jun 23, 2020
6b9825e
Marking only SparkSession as serializable
Niharikadutta Jun 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Marking only SparkSession as serializable
  • Loading branch information
@Niharikadutta
Niharikadutta committed Jun 23, 2020
commit 6b9825e5f93161e36afb0c6f8921efbc124ecd0a
4 changes: 0 additions & 4 deletions src/csharp/Microsoft.Spark/Interop/Ipc/JvmBridge.cs
10000 View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ namespace Microsoft.Spark.Interop.Ipc
/// Using a concurrent socket connection queue (lightweight synchronization mechanism)
/// supporting async JVM calls like StreamingContext.AwaitTermination()
/// </summary>
[Serializable]
internal sealed class JvmBridge : IJvmBridge
{
// TODO: On .NET Core 2.1, Span<object> could be used with a stack-based
Expand All @@ -29,13 +28,10 @@ internal sealed class JvmBridge : IJvmBridge
[ThreadStatic]
private static object[] s_twoArgArray;
[ThreadStatic]
[NonSerialized]
private static MemoryStream s_payloadMemoryStream;

[NonSerialized]
private readonly ConcurrentQueue<ISocketWrapper> _sockets =
new ConcurrentQueue<ISocketWrapper>();
[NonSerialized]
private readonly ILoggerService _logger =
LoggerServiceFactory.GetLogger(typeof(JvmBridge));
private readonly int _portNumber;
Expand Down
2 changes: 0 additions & 2 deletions src/csharp/Microsoft.Spark/Interop/Ipc/JvmObjectReference.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ namespace Microsoft.Spark.Interop.Ipc
/// The reason for having another layer on top of string id is
/// so that JvmObjectReference can be copied.
/// </summary>
[Serializable]
internal sealed class JvmObjectId
{
private static readonly ILoggerService s_logger =
Expand Down Expand Up @@ -113,7 +112,6 @@ internal interface IJvmObjectReferenceProvider
/// <summary>
/// Reference to object created in JVM.
/// </summary>
[Serializable]
internal sealed class JvmObjectReference : IJvmObjectReferenceProvider
{
/// <summary>
Expand Down
1 change: 1 addition & 0 deletions src/csharp/Microsoft.Spark/Sql/SparkSession.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ namespace Microsoft.Spark.Sql
[Serializable]
public sealed class SparkSession : IDisposable, IJvmObjectReferenceProvider
{
[NonSerialized]
private readonly JvmObjectReference _jvmObject;

[NonSerialized]
Expand Down
0