Add chunked transfer-encoding

apl-package.json

@@ -19,5 +19,5 @@
   source: "source/HttpCommand.dyalog",
   tags: "mac-os,windows,linux,http,dyalog,pi",
   userCommandScript: "",
-  version: "5.4.6",
+  version: "5.8.0",
 }

docs/release-notes.md

@@ -1,3 +1,5 @@
+## Version 5.8
+* Added configuration setting [`ChunkSize`](./request-settings.md#chunksize) to enable sending request payloads using "chunked" transfer-encoding.
 ## Version 5.7
 * Added shared setting [HeaderSubstitution](./request-settings.md#headersubstitution) to control if and how environment variables are injected into header names and/or values.
 ## Version 5.6

docs/request-settings.md

@@ -220,6 +220,14 @@ If <code>AuthType</code> is not set and <code>Auth</code> is set to either a cha
 <code> logged_in  github.com   05-AUG-2022</code><br/>  
 </td></tr></table>
 
+### `ChunkSize`
+
+|--|--|
+| Description | A non-zero `ChunkSize` will make `HttpCommand` use "chunked" transfer-encoding and send the payload of the request in chunk of `ChunkSize` bytes. |
+| Default | `0` meaning do not use "chunked" transfer-encoding |
+| Details | Using a non-zero `ChunkSize` will cause `HttpCommand` to format the payload of the request according to the specification for "chunked" transfer-encoding. This involves breaking the payload into `ChunkSize`-sized chunks each preceded by the hexadecimal length of the chunk. <br><br>**Note:** In the current implementation, the entire, reformatted, payload is sent in a single request to the host. | 
+
+
 ## Shared Settings
 ### `HeaderSubstitution`
 <table><tr>

mkdocs.yml

@@ -46,6 +46,8 @@ theme:
 
 extra: 
   generator: false
+  version:
+    provider: mike
 
 extra_css:
   - css/main.css
@@ -76,6 +78,10 @@ markdown_extensions:
   - abbr
   - footnotes
   - attr_list 
+  - def_list
+  - markdown_tables_extended
+  - pymdownx.tasklist:
+      custom_checkbox: true 
   - pymdownx.emoji:
       emoji_index: !!python/name:materialx.emoji.twemoji
       emoji_generator: !!python/name:materialx.emoji.to_svg

source/HttpCommand.dyalog

@@ -7,7 +7,7 @@
     ∇ r←Version
     ⍝ Return the current version
       :Access public shared
-      r←'HttpCommand' '5.7.0' '2024-05-17'
+      r←'HttpCommand' '5.8.0' '2024-07-17'
     ∇
 
 ⍝ Request-related fields
@@ -20,6 +20,7 @@
     :field public Auth←''                          ⍝ authentication string
     :field public AuthType←''                      ⍝ authentication type
     :field public BaseURL←''                       ⍝ base URL to use when making multiple requests to the same host
+    :field public ChunkSize←0                      ⍝ set to size of chunk if using chunked transfer encoding
     :field public shared HeaderSubstitution←''     ⍝ delimiters to indicate environment/configuration settings be substituted in headers, set to '' to disable
 
 
@@ -470,6 +471,7 @@
           hdrs←'User-Agent'(hdrs addHeader)deb'Dyalog-',1↓∊'/',¨2↑Version
           hdrs←'Accept'(hdrs addHeader)'*/*'
           hdrs←'Accept-Encoding'(hdrs addHeader)'gzip, deflate'
+
           :If ~0∊⍴Auth
               :If (1<|≡Auth)∨':'∊Auth ⍝ (userid password) or userid:password
               :AndIf ('basic'≡lc AuthType)∨0∊⍴AuthType
@@ -478,13 +480,20 @@
               :EndIf
               hdrs←'Authorization'(hdrs setHeader)deb AuthType,' ',⍕Auth
           :EndIf
+
           :If '∘???∘'≡hdrs getHeader'cookie' ⍝ if the user has specified a cookie header, it takes precedence
           :AndIf ~0∊⍴cookies←r applyCookies Cookies
               hdrs←'Cookie'(hdrs addHeader)formatCookies cookies
           :EndIf
+
           :If ~0∊⍴auth
               hdrs←'Authorization'(hdrs addHeader)auth
           :EndIf
+
+          :If 0≠ChunkSize
+              hdrs←'Transfer-Encoding'(hdrs addHeader)'chunked'
+          :EndIf
+
           :If proxied
               :If ~0∊⍴ProxyAuth
                   :If (1<|≡ProxyAuth)∨':'∊ProxyAuth ⍝ (userid password) or userid:password
@@ -581,13 +590,18 @@
               :EndSelect
 
               :If RequestOnly>SuppressHeaders ⍝ Conga supplies content-length, but for RequestOnly we need to insert it
+              :AndIf 0=ChunkSize
                   hdrs←'Content-Length'(hdrs addHeader)⍕⍴parms
               :EndIf
           :EndIf
       :EndIf
 
       hdrs⌿⍨←~0∊¨≢¨hdrs[;2] ⍝ remove any headers with empty values
 
+      :If 0≠ChunkSize
+          parms←ChunkSize Chunk parms
+      :EndIf
+
       :If RequestOnly
           r←cmd,' ',(path,(0∊⍴urlparms)↓'?',urlparms),' HTTP/1.1',(∊{NL,⍺,': ',∊⍕⍵}/privatize environment hdrs),NL,NL,parms
           →∆EXIT
@@ -926,6 +940,19 @@
      ∆EXIT:
     ∇
 
+    ∇ r←size Chunk payload;l;n;last;lens;hlens;mask
+      :Access public shared
+    ⍝ Split payload into chunks for chunked transfer-encoding
+      l←≢payload ⍝ payload length
+      n←⌊l÷size  ⍝ number of whole chunk
+      last←l-n×size ⍝ size of last chunk
+      lens←(n⍴size),last,(last≠0)/0 ⍝ chunk lengths + 0 for terminating chunk
+      hlens←d2h¨lens ⍝ hex lengths
+      mask←0 1 0(⊢⊢⍤/(⍴⊢)⍴⊣),(2+≢¨hlens),lens,[1.1]2 ⍝ expansion mask
+      r←mask\payload ⍝ expand payload
+      r[⍸~mask]←2⌽∊NL∘,¨hlens,¨⊂NL ⍝ insert chunk information
+    ∇
+
     ∇ rc←r CheckPayloadSize size
     ⍝ checks if payload exceeds MaxPayloadSize
       rc←0
@@ -1027,7 +1054,7 @@
     splitOnFirst←{(⍺↑⍨¯1+p)(⍺↓⍨p←⌊/⍺⍳⍵)} ⍝ split ⍺ on first occurrence of ⍵ (removing first ⍵)
     splitOn←≠⊆⊣ ⍝ split ⍺ on all ⍵ (removing ⍵)
     h2d←{⎕IO←0 ⋄ 16⊥'0123456789abcdef'⍳lc ⍵} ⍝ hex to decimal
-    d2h←{⎕IO←0 ⋄ '0123456789ABCDEF'[16(⊥⍣¯1)⍵]} ⍝ decimal to hex
+    d2h←{⎕IO←0 ⋄ '0123456789ABCDEF'[((1∘⌈≢)↑⊢)16(⊥⍣¯1)⍵]} ⍝ decimal to hex
     getchunklen←{¯1=len←¯1+⊃(NL⍷⍵)/⍳⍴⍵:¯1 ¯1 ⋄ chunklen←h2d len↑⍵ ⋄ (⍴⍵)<len+chunklen+4:¯1 ¯1 ⋄ len chunklen}
     toInt←{0∊⍴⍵:⍬ ⋄ ~3 5∊⍨10|⎕DR t←1⊃2⊃⎕VFI ⍕⍵:⍬ ⋄ t≠⌊t:⍬ ⋄ t} ⍝ simple char to int
     fmtHeaders←{0∊⍴⍵:'' ⋄ (firstCaps¨⍵[;1])(,∘⍕¨⍵[;2])} ⍝ formatted HTTP headers