distributed query works. Changed YAML also

wilkinsonlab · Dec 29, 2023 · dc2309a · dc2309a
1 parent 7c3b398
commit dc2309a
Show file tree

Hide file tree

Showing 11 changed files with 142 additions and 95 deletions.
diff --git a/VP/interfaces/administrative-sparql.yaml b/VP/interfaces/administrative-sparql.yaml
@@ -100,31 +100,7 @@ paths:
           required: true
           schema:
             type: string
-            example: select * where {?s ?p ?o} limit 10
+            example: SELECT DISTINCT ?type WHERE {?s a ?type}
       responses:
         '200':
           $ref: "#/components/responses/200SparqlResult"
-    post:
-      tags:
-        - SPARQL
-      summary: Execute SPARQL query
-      description: |
-        Execute a SPARQL query on the repository. The result format is based on the type of result (boolean, variable bindings, or RDF data) and the negotiated acceptable content-type. Note that RDF4J supports executing SPARQL queries with either a GET or a POST request. POST is supported for queries that are too large to be encoded as a query parameter.
-      requestBody:
-        description: the SPARQL query to execute
-        required: true
-        content:
-          application/sparql-query:
-            schema:
-              type: string
-            example:
-              "SELECT * WHERE {?S ?P ?O } LIMIT 10"
-          application/x-www-form-urlencoded:
-            schema:
-              type: string
-            example:
-              query=SELECT%20%2A%20WHERE%20%7B%3FS%20%3FP%20%3FO%20%7D%20LIMIT%2010
-      responses:
-        '200':
-          $ref: "#/components/responses/200SparqlResult"
-
diff --git a/VP/interfaces/fdp-sparql.yaml b/VP/interfaces/fdp-sparql.yaml
@@ -101,31 +101,7 @@ paths:
           required: true
           schema:
             type: string
-            example: select * where {?s ?p ?o} limit 10
+            example: SELECT DISTINCT ?type WHERE {?s a ?type}
       responses:
         '200':
           $ref: "#/components/responses/200SparqlResult"
-    post:
-      tags:
-        - SPARQL
-      summary: Execute SPARQL query
-      description: |
-        Execute a SPARQL query on the repository. The result format is based on the type of result (boolean, variable bindings, or RDF data) and the negotiated acceptable content-type. Note that RDF4J supports executing SPARQL queries with either a GET or a POST request. POST is supported for queries that are too large to be encoded as a query parameter.
-      requestBody:
-        description: the SPARQL query to execute
-        required: true
-        content:
-          application/sparql-query:
-            schema:
-              type: string
-            example:
-              "SELECT * WHERE {?S ?P ?O } LIMIT 10"
-          application/x-www-form-urlencoded:
-            schema:
-              type: string
-            example:
-              query=SELECT%20%2A%20WHERE%20%7B%3FS%20%3FP%20%3FO%20%7D%20LIMIT%2010
-      responses:
-        '200':
-          $ref: "#/components/responses/200SparqlResult"
-
diff --git a/VP/interfaces/location-sparql.yaml b/VP/interfaces/location-sparql.yaml
@@ -101,31 +101,7 @@ paths:
           required: true
           schema:
             type: string
-            example: select * where {?s ?p ?o} limit 10
+            example: SELECT DISTINCT ?type WHERE {?s a ?type}
       responses:
         '200':
           $ref: "#/components/responses/200SparqlResult"
-    post:
-      tags:
-        - SPARQL
-      summary: Execute SPARQL query
-      description: |
-        Execute a SPARQL query on the repository. The result format is based on the type of result (boolean, variable bindings, or RDF data) and the negotiated acceptable content-type. Note that RDF4J supports executing SPARQL queries with either a GET or a POST request. POST is supported for queries that are too large to be encoded as a query parameter.
-      requestBody:
-        description: the SPARQL query to execute
-        required: true
-        content:
-          application/sparql-query:
-            schema:
-              type: string
-            example:
-              "SELECT * WHERE {?S ?P ?O } LIMIT 10"
-          application/x-www-form-urlencoded:
-            schema:
-              type: string
-            example:
-              query=SELECT%20%2A%20WHERE%20%7B%3FS%20%3FP%20%3FO%20%7D%20LIMIT%2010
-      responses:
-        '200':
-          $ref: "#/components/responses/200SparqlResult"
-
diff --git a/VP/vp-interface/Gemfile b/VP/vp-interface/Gemfile
@@ -30,6 +30,7 @@ gem "uri", "~>0.12.2"
 gem "xml-simple", "~>1.1.8"
 gem "fileutils", "~>1.5.0"
 gem 'openapi3_parser', '~> 0.9.2'
+gem "cgi", "~> 0.4.1"
 
 group :test do
   gem "capybara"

diff --git a/VP/vp-interface/Gemfile.lock b/VP/vp-interface/Gemfile.lock
@@ -32,6 +32,7 @@ GEM
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
+    cgi (0.4.1)
     coderay (1.1.3)
     commonmarker (0.23.10)
     concurrent-ruby (1.2.2)
@@ -366,6 +367,7 @@ PLATFORMS
 DEPENDENCIES
   bcrypt
   capybara
+  cgi (~> 0.4.1)
   debase!
   fileutils (~> 1.5.0)
   json (~> 2.7.1)

diff --git a/VP/vp-interface/app/controllers/application_controller.rb b/VP/vp-interface/app/controllers/application_controller.rb
@@ -4,6 +4,7 @@
 require "sinatra"
 require "sinatra/base"
 require "json"
+require "erb"
 # DO NOT change the order of loading below.  The files contain executable code that builds the overall configuration before this module starts
 require_relative "configuration"  # VPConfig and FDPConfig
 require_relative "models"

diff --git a/VP/vp-interface/app/controllers/routes.rb b/VP/vp-interface/app/controllers/routes.rb
@@ -64,6 +64,13 @@ def set_routes(classes: allclasses)
     erb :wordcloud
   end
 
+  post "/flair-gg-vp-server/execute-data-services" do
+
+    @results = VP.current_vp.execute_data_services(params: params)
+    erb :execution_results_layout
+
+  end
+
   before do
     @services = VP.current_vp.collect_data_services
 

diff --git a/VP/vp-interface/app/views/execution_results_layout.erb b/VP/vp-interface/app/views/execution_results_layout.erb
@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+<!--[if lt IE 7]> <html class="no-js ie6 oldie" lang="en"> <![endif]-->
+<!--[if IE 7]>    <html class="no-js ie7 oldie" lang="en"> <![endif]-->
+<!--[if IE 8]>    <html class="no-js ie8 oldie" lang="en"> <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->
+
+  <%= erb :_head %>
+
+  <body>
+    <div class="wrapper">
+    <div><a href="/flair-gg-vp-server/resources"><img src="/images/flair-gg-logo.png" width="400px"/></a></div>
+
+
+    <%= erb :_sidebar %>
+
+      <h2>Results</h2>
+
+
+          <% @results.each_key do |site| %>
+            <table>
+              <tr>
+                <td colspan=2>
+                  <h3>SOURCE: <a href="http://<%= site %>" _target="_blank"><%= site %></a></h3>
+                </td>
+              </tr>
+              <tr>
+                <td>&nbsp;&nbsp;&nbsp;&nbsp;</td>
+                  <td>
+                      <textarea rows="30" cols="65"><%= @results[site] %></textarea>       
+                  </td>
+                </tr>
+            </table>
+            <br/><br/>
+          <% end %>
+
+
+    <%= erb :_footer %>
+    </div>
+
+    <script src="http://ajax.googleapis.com/ajax/libs/jquery/1/jquery.min.js"></script>
+    <!--[if lt IE 7]>
+      <script src="//ajax.googleapis.com/ajax/libs/chrome-frame/1.0.2/CFInstall.min.js"></script>
+      <script>window.attachEvent("onload",function(){CFInstall.check({mode:"overlay"})})</script>
+    <![endif]-->
+  </body>
+</html>
diff --git a/VP/vp-interface/app/views/services_layout.erb b/VP/vp-interface/app/views/services_layout.erb
@@ -13,15 +13,15 @@
 
       <%= erb :_sidebar %>
 
-        <h2>Common Parameters</h2>
+        <h3>Service Execution Parameters</h3>
 
 <%
 =begin
 
 class ServiceCollection  @servicecollection
-  attr_accessor :vpgraph, :servicetype, :allservices, :warnings
+  attr_accessor :vpgraph, :servicetype, :allservices, :warnings, :escapedtype
 class Service  (from allservices)
-  attr_accessor :contact, :title, :openapi, :endpoint, :apiobject, :base, :paths
+  attr_accessor :contact, :title, :openapi, :endpoint, :apiobject, :base, :paths,  :escapedendpoint
   retrieve_parameters:  params[param.name] = param  # responds to "in", "name", "description", "schema.example", "schema.type"
 
 @commonparameters  (shared by all services) commonparams[paramname] = params[paramname], 
@@ -30,16 +30,47 @@ every param  responds to "in", "name", "description", "schema.example", "schema.
 =end
 %>
 
-        <form action = "/flair-gg-vp-server/execute-data-services">
+        <form action = "/flair-gg-vp-server/execute-data-services" method="post">
+        <input type="hidden" name="servicetype" value="<%= @servicecollection.escapedtype %>">
 
         <% @commonparams.each do |paramname, param| %>
-          <label for="<%= paramname %>" style="font-size: .75em;"><%= paramname %></label>
-          <input type="text" name="<%= paramname %>" id="<%= paramname %>"><br/><br/>
+          <label for="<%= paramname %>" style="font-size: .75em;"><%= paramname %></label><br/>
+          <input type="text" size="70" name="<%= paramname %>" id="<%= paramname %>" value="<%= param.schema.example %>"><br/><br/>
         <% end %>
 
+        <hr>
+        <br/><br/>
 
-
-
+        <h3>Which providers should these values be sent to?</h3>
+        <% if @servicecollection.warnings.first %>
+          WARNINGS:<br/><br/>
+          <% @servicecollection.warnings.each do |warn| %>
+            - <%= warn %><br/>
+          <% end %>
+        <% end %>
+
+        <table>
+          <% @servicecollection.allservices.each do |service| %>
+            <tr>
+              <td style="padding-right: 5px;">
+                <input type="checkbox" name="endpoint[]" value="<%= service.escapedendpoint %>" id="<%= service.escapedendpoint %>"/>
+              </td>
+              <td>
+                <h4>Provider: <a href="<%= service.contact %>" _target="_blank"><%= service.contact %></a></h4>
+              </td>
+            </tr>
+            <tr>
+              <td></td>
+              <td><span style="font-size: .75em;"><%= service.title %></span></td>
+            </tr>
+            <tr>
+              <td>---<br/></td><td></td>
+            </tr>
+          <% end %>
+        </table>
+        <br/><br/>
+        <input type="submit" value="Let's do this!"/>
+        <form>
 
 
       <%= erb :_footer %>

diff --git a/VP/vp-interface/lib/services.rb b/VP/vp-interface/lib/services.rb
@@ -1,19 +1,21 @@
 require "openapi3_parser"
+require "cgi"
 
 class ServiceCollection
-  attr_accessor :vpgraph, :servicetype, :allservices, :warnings
+  attr_accessor :vpgraph, :servicetype, :allservices, :warnings, :escapedtype
 
   def initialize(vpgraph:, servicetype:)
     @vpgraph = vpgraph
     @servicetype = servicetype
+    # @escapedtype = ERB::Util.url_encode(servicetype)
+    @escapedtype = CGI.escape(servicetype)
     @allservices = []
     @warnings = []
 
     collect_similar_services
   end
 
   def collect_similar_services
-
     warn "in collect similar services"
     vpd = SPARQL.parse("
     #{VP::NAMESPACES}
@@ -35,7 +37,12 @@ def collect_similar_services
     results = vpgraph.query(vpd)
     results.each do |result|
       service = Service.new(contact: result[:contact], title: result[:title], openapi: result[:openapi], endpoint: result[:endpoint])
-      @allservices << service
+      if service.successful
+        @allservices << service  # only if there's a match!
+      else
+        # if not, no path in the openapi YAML matched the path in the DCAT
+        @warnings << "The endpoint for #{service.contact} (#{service.endpoint}) in the DCAT did not match any endpoint in the OpenAPI YAML. It was therefore not returned"
+      end
     end
   end
 
@@ -48,7 +55,7 @@ def gather_common_parameters
           warn "found #{paramname}"
           if commonparams[paramname]
             warn "already found #{paramname} - now overwriting"
-            @warnings << "overwriting with #{fullpath} - this might be good news!?"
+            # @warnings << "overwriting with #{fullpath} - this might be good news!?"
           end
           commonparams[paramname] = params[paramname]
         end
@@ -59,30 +66,43 @@ def gather_common_parameters
 end
 
 class Service
-  attr_accessor :contact, :title, :openapi, :endpoint, :apiobject, :base, :paths
+  attr_accessor :contact, :title, :openapi, :endpoint, :apiobject, :base, :paths, :escapedendpoint, :successful
 
   def initialize(contact:, title:, openapi:, endpoint:)
     @title = title
     @openapi = openapi
     @endpoint = endpoint
+#    @escapedendpoint = ERB::Util.url_encode(endpoint)
+    @escapedendpoint = CGI.escape(endpoint)
     @contact = contact
     @paths = {}
-    @apiobject = retrieve_endpoint(openapi: @openapi)
+    @successful = false
+    @apiobject = retrieve_endpoint(openapi: @openapi) # this will fill the @paths with a [:get] and [:post]
   end
 
   def retrieve_endpoint(openapi:)
     warn "retrieving #{openapi}"
     api = Openapi3Parser.load_url(openapi)
-    #@base = api["servers"].first[a.paths.keys"url"]
     api.paths.each do |path, pathitem|
       warn "path #{path}"
       base = pathitem.servers.first.url
-      get = pathitem.get
-      post = pathitem.post
+      get = pathitem.get  #  Openapi3Parser::Node::Operation or nil
+      post = pathitem.post  #  Openapi3Parser::Node::Operation or nil
       fullpath = base + path
-      @paths[fullpath] = {} unless @paths[fullpath]
+      warn "testing #{fullpath} against #{@endpoint}|" 
+      unless fullpath == @endpoint  # this seems a bit dangerous, but it should be the same as what is in the DCAT record...
+        warn "TEST FAILED"
+        next
+      end
+
+      @paths[fullpath] = {} unless @paths[fullpath]  # initialize
       @paths[fullpath] = { get: get, post: post }
+      if fullpath == @endpoint # there can be only one match!  Match with the dcat record
+        @successful = true
+        break
+      end
     end
+    api
   end
 
   def retrieve_parameters(fullpath:, operation:)

diff --git a/VP/vp-interface/lib/vp.rb b/VP/vp-interface/lib/vp.rb
@@ -233,8 +233,8 @@ def collect_data_services
     results = @graph.query(vpd)
     prehash = {}
     results.each do |r|
-      type = r[:type]; kw = r[:kw]; s = r[:s]
-      warn "subject #{s} type #{type} kw #{kw}"
+      type = r[:type]; kw = r[:kw];
+      warn "subject type #{type} kw #{kw}"
       prehash[type] = "" unless prehash[type]
       prehash[type] += "#{kw}, "
     end
@@ -249,6 +249,17 @@ def retrieve_sevices(term:)
     [servicecollection, commonparams]
   end
 
+  def execute_data_services(params:)
+    endpoints = params.delete("endpoint")
+    results = {}
+    endpoints.each do |ep|
+      endpoint = CGI.unescape(ep)
+      result = RestClient.get(endpoint, {params: params})
+      warn result.inspect
+      results[endpoint] = result.body
+    end
+    results
+  end
 
   def match_type_to_icon(type:)
     t = type.match(%r{[\#/](\w+?)$})[1].downcase.to_sym