8000 Support Link Header pagination in WebCmdlets by SteveL-MSFT · Pull Request #3828 · PowerShell/PowerShell · GitHub
[go: up one dir, main page]
Skip to content

Support Link Header pagination in WebCmdlets #3828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 24, 2017
Merged

Support Link Header pagination in WebCmdlets #3828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
741d87f
Support Link Header pagination in WebCmdlets to make it easier for th…
SteveL-MSFT Apr 29, 2017
283f31a
removed unnecessary refs to namespaces
SteveL-MSFT May 19, 2017
cbf2afb
addressed code review feedback
May 23, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions ...hell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,29 @@ public override string CustomMethod
set { base.CustomMethod = value; }
}

/// <summary>
/// enable automatic following of rel links
/// </summary>
[Parameter]
[Alias("FL")]
public SwitchParameter FollowRelLink
{
get { return base._followRelLink; }
set { base._followRelLink = value; }
}

/// <summary>
/// gets or sets the maximum number of rel links to follow
/// </summary>
[Parameter]
[Alias("ML")]
[ValidateRange(1, Int32.MaxValue)]
public int MaximumFollowRelLink
{
get { return base._maximumFollowRelLink; }
set { base._maximumFollowRelLink = value; }
}

#endregion Parameters

#region Helper Methods
Expand Down
9 changes: 9 additions & 0 deletions ...ll.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,14 @@ public class InvokeWebRequestCommand : WebRequestPSCmdlet
{
#region Virtual Method Overrides

/// <summary>
/// Default constructor for InvokeWebRequestCommand
/// </summary>
public InvokeWebRequestCommand() : base()
{
this._parseRelLink = true;
}

/// <summary>
/// Process the web response and output corresponding objects.
/// </summary>
Expand All @@ -46,6 +54,7 @@ internal override void ProcessResponse(HttpResponseMessage response)
// creating a MemoryStream wrapper to response stream here to support IsStopping.
responseStream = new WebResponseContentMemoryStream(responseStream, StreamHelper.ChunkSize, this);
WebResponseObject ro = WebResponseObjectFactory.GetResponseObject(response, responseStream, this.Context, UseBasicParsing);
ro.RelationLink = _relationLink;
WriteObject(ro);

// use the rawcontent stream from WebResponseObject for further
Expand Down
229 changes: 158 additions & 71 deletions ...erShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebRequestPSCmdlet.CoreClr.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
using System.Security.Cryptography;
using System.Threading;
using System.Xml;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Linq;

namespace Microsoft.PowerShell.Commands
{
Expand Down Expand Up @@ -61,6 +64,26 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet
/// </summary>
private CancellationTokenSource _cancelToken = null;

/// <summary>
/// Parse Rel Links
/// </summary>
internal bool _parseRelLink = false;

/// <summary>
/// Automatically follow Rel Links
/// </summary>
internal bool _followRelLink = false;

/// <summary>
/// Automatically follow Rel Links
/// </summary>
internal Dictionary<string, string> _relationLink = null;

/// <summary>
/// Maximum number of Rel Links to follow
/// </summary>
internal int _maximumFollowRelLink = Int32.MaxValue;

private HttpMethod GetHttpMethod(WebRequestMethod method)
{
switch (Method)
Expand Down Expand Up @@ -234,7 +257,7 @@ internal virtual HttpRequestMessage GetRequest(Uri uri)
}

// Some web sites (e.g. Twitter) will return exception on POST when Expect100 is sent
// Default behaviour is continue to send body content anyway after a short period
// Default behavior is continue to send body content anyway after a short period
// Here it send the two part as a whole.
request.Headers.ExpectContinue = false;

Expand Down Expand Up @@ -374,91 +397,121 @@ protected override void ProcessRecord()
PrepareSession();

using (HttpClient client = GetHttpClient())
using (HttpRequestMessage request = GetRequest(Uri))
{
FillRequestStream(request);
try
int followedRelLink = 0;
Uri uri = Uri;
do
{
long requestContentLength = 0;
if (request.Content != null)
requestContentLength = request.Content.Headers.ContentLength.Value;

string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture,
"{0} {1} with {2}-byte payload",
request.Method,
request.RequestUri,
requestContentLength);
WriteVerbose(reqVerboseMsg);

HttpResponseMessage response = GetResponse(client, request);

string contentType = ContentHelper.GetContentType(response);
string respVerboseMsg = string.Format(CultureInfo.CurrentCulture,
"received {0}-byte response of content type {1}",
response.Content.Headers.ContentLength,
contentType);
WriteVerbose(respVerboseMsg);

if (!response.IsSuccessStatusCode)
if (followedRelLink > 0)
{
string message = String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.ResponseStatusCodeFailure,
(int)response.StatusCode, response.ReasonPhrase);
HttpResponseException httpEx = new HttpResponseException(message, response);
ErrorRecord er = new ErrorRecord(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request);
string detailMsg = "";
StreamReader reader = null;
string linkVerboseMsg = string.Format(CultureInfo.CurrentCulture,
WebCmdletStrings.FollowingRelLinkVerboseMsg,
uri.AbsoluteUri);
WriteVerbose(linkVerboseMsg);
}

using (HttpRequestMessage request = GetRequest(uri))
{
FillRequestStream(request);
try
{
reader = new StreamReader(StreamHelper.GetResponseStream(response));
// remove HTML tags making it easier to read
detailMsg = System.Text.RegularExpressions.Regex.Replace(reader.ReadToEnd(), "<[^>]*>","");
}
catch (Exception)
{
// catch all
}
finally
{
if (reader != null)
long requestContentLength = 0;
if (request.Content != null)
requestContentLength = request.Content.Headers.ContentLength.Value;

string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture,
WebCmdletStrings.WebMethodInvocationVerboseMsg,
request.Method,
request.RequestUri,
requestContentLength);
WriteVerbose(reqVerboseMsg);

HttpResponseMessage response = GetResponse(client, request);

string contentType = ContentHelper.GetContentType(response);
string respVerboseMsg = string.Format(CultureInfo.CurrentCulture,
WebCmdletStrings.WebResponseVerboseMsg,
response.Content.Headers.ContentLength,
contentType);
WriteVerbose(respVerboseMsg);

if (!response.IsSuccessStatusCode)
{
reader.Dispose();
string message = String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.ResponseStatusCodeFailure,
(int)response.StatusCode, response.ReasonPhrase);
HttpResponseException httpEx = new HttpResponseException(message, response);
ErrorRecord er = new ErrorRecord(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request);
string detailMsg = "";
StreamReader reader = null;
try
{
reader = new StreamReader(StreamHelper.GetResponseStream(response));
// remove HTML tags making it easier to read
detailMsg = System.Text.RegularExpressions.Regex.Replace(reader.ReadToEnd(), "<[^>]*>","");
}
catch (Exception)
{
// catch all
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
if (!String.IsNullOrEmpty(detailMsg))
{
er.ErrorDetails = new ErrorDetails(detailMsg);
}
ThrowTerminatingError(er);
}

if (_parseRelLink || _followRelLink)
{
ParseLinkHeader(response, uri);
}
ProcessResponse(response);
UpdateSession(response);

// If we hit our maximum redirection count, generate an error.
// Errors with redirection counts of greater than 0 are handled automatically by .NET, but are
// impossible to detect programmatically when we hit this limit. By handling this ourselves
// (and still writing out the result), users can debug actual HTTP redirect problems.
if (WebSession.MaximumRedirection == 0) // Indicate "HttpClientHandler.AllowAutoRedirect == false"
{
if (response.StatusCode == HttpStatusCode.Found ||
response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.MovedPermanently)
{
ErrorRecord er = new ErrorRecord(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request);
er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded);
WriteError(er);
}
}
}
if (!String.IsNullOrEmpty(detailMsg))
catch (HttpRequestException ex)
{
er.ErrorDetails = new ErrorDetails(detailMsg);
ErrorRecord er = new ErrorRecord(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request);
if (ex.InnerException != null)
{
er.ErrorDetails = new ErrorDetails(ex.InnerException.Message);
}
ThrowTerminatingError(er);
}
ThrowTerminatingError(er);
}

ProcessResponse(response);
UpdateSession(response);

// If we hit our maximum redirection count, generate an error.
// Errors with redirection counts of greater than 0 are handled automatically by .NET, but are
// impossible to detect programmatically when we hit this limit. By handling this ourselves
// (and still writing out the result), users can debug actual HTTP redirect problems.
if (WebSession.MaximumRedirection == 0) // Indicate "HttpClientHandler.AllowAutoRedirect == false"
{
if (response.StatusCode == HttpStatusCode.Found ||
response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.MovedPermanently)
if (_followRelLink)
{
ErrorRecord er = new ErrorRecord(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request);
er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded);
WriteError(er);
if (!_relationLink.ContainsKey("next"))
{
return;
}
uri = new Uri(_relationLink["next"]);
followedRelLink++;
}
}
}
catch (HttpRequestException ex)
{
ErrorRecord er = new ErrorRecord(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request);
if (ex.InnerException != null)
{
er.ErrorDetails = new ErrorDetails(ex.InnerException.Message);
}
ThrowTerminatingError(er);
}
while (_followRelLink && (followedRelLink < _maximumFollowRelLink));
}
}
catch (CryptographicException ex)
Expand Down Expand Up @@ -619,6 +672,40 @@ internal long SetRequestContent(HttpRequestMessage request, IDictionary content)

}

internal void ParseLinkHeader(HttpResponseMessage response, System.Uri requestUri)
{
if (_relationLink == null)
{
_relationLink = new Dictionary<string, string>();
}
else
{
_relationLink.Clear();
}

// we only support the URL in angle brackets and `rel`, other attributes are ignored
// user can still parse it themselves via the Headers property
string pattern = "<(?<url>.*?)>;\\srel=\"(?<rel>.*?)\"";
IEnumerable<string> links;
if (response.Headers.TryGetValues("Link", out links))
{
foreach(string link in links.FirstOrDefault().Split(","))
{
Match match = Regex.Match(link, pattern);
if (match.Success)
{
string url = match.Groups["url"].Value;
string rel = match.Groups["rel"].Value;
if (url != String.Empty && rel != String.Empty && !_relationLink.ContainsKey(rel))
{
Uri absoluteUri = new Uri(requestUri, url);
_relationLink.Add(rel, absoluteUri.AbsoluteUri.ToString());
}
}
}
}
}

#endregion Helper Methods
}
}
Expand Down
5 changes: 5 additions & 0 deletions ...werShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseObject.CoreClr.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ public Dictionary<string, IEnumerable<string>> Headers
}
}

/// <summary>
/// gets the RelationLink property
/// </summary>
public Dictionary<string, string> RelationLink { get; internal set; }

#endregion

#region Constructors
Expand Down
11 changes: 10 additions & 1 deletion src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,4 +213,13 @@
<data name="ResponseStatusCodeFailure" xml:space="preserve">
<value>Response status code does not indicate success: {0} ({1}).</value>
</data>
</root>
<data name="FollowingRelLinkVerboseMsg" xml:space="preserve">
<value>Following rel link {0}</value>
</data>
<data name="WebMethodInvocationVerboseMsg" xml:space="preserve">
<value>{0} {1} with {2}-byte payload</value>
</data>
<data name="WebResponseVerboseMsg" xml:space="preserve">
<value>received {0}-byte response of content type {1}</value>
</data>
</root>
2 changes: 2 additions & 0 deletions ...commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1271,6 +1271,7 @@ private static IEnumerable<FormatViewDefinition> ViewsOf_Microsoft_PowerShell_Co
.AddItemProperty(@"Links")
.AddItemProperty(@"ParsedHtml")
.AddItemProperty(@"RawContentLength")
.AddItemProperty(@"RelationLink")
.EndEntry()
.EndList());
}
Expand All @@ -1291,6 +1292,7 @@ private static IEnumerable<FormatViewDefinition> ViewsOf_Microsoft_PowerShell_Co
", label: "RawContent")
.AddItemProperty(@"Headers")
.AddItemProperty(@"RawContentLength")
.AddItemProperty(@"RelationLink")
.EndEntry()
.EndList());
}
Expand Down
Loading
0