[fix] Encodig

malina · Mar 6, 2017 · b4b0470 · b4b0470
1 parent 824b393
commit b4b0470
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 19 deletions.
diff --git a/src/metascraper/parser.cr b/src/metascraper/parser.cr
@@ -18,10 +18,11 @@ module Metascraper
       @url = url
       @response = get_request(url).as(HTTP::Client::Response)
 
-      response_body = encode_body
+      response_body = @response.body
 
       @document = XML.parse_html(response_body)
-      @texts = Parsers::Text.new(@document)
+      get_charset()
+      @texts = Parsers::Text.new(@document, config)
       @images = Parsers::Images.new(@document, config)
       @videos = config.skip_video ? Videos.new : Parsers::Videos.new(@document, config)
     end
@@ -35,18 +36,28 @@ module Metascraper
       response
     end
 
-    private def encode_body : String
-      charset = @response.charset.as(String) rescue Metascraper::Config::DEFAULT_CHARSET
+    private def get_charset : Void
+      from_response_charset = @response.charset.as(String)
 
-      if charset == config.charset
-        @response.body
+      unless charset_from_html.empty?
+        config.charset = charset_from_html
       else
-        config.charset = charset
-        Utils.new(
-          @response.body,
-          charset
-        ).encodeToUtf8.as(String)
+        config.charset = from_response_charset
       end
     end
+
+    private def charset_from_html : String
+      meta = @document.xpath_node("//meta[contains(@content, 'charset')]/@content")
+      if meta
+        value = meta.content
+        substring = "charset="
+        index = value.index(substring).as(Int32)
+        value[(index+substring.size)..(value.size-1)]
+      else
+        ""
+      end
+    rescue
+      ""
+    end
   end
 end
diff --git a/src/metascraper/parsers/text_parser.cr b/src/metascraper/parsers/text_parser.cr
@@ -1,23 +1,24 @@
 module Metascraper
   module Parsers
     class Text
-      getter document
-      def initialize(@document : XML::Node)
+      getter document, config
+      def initialize(@document : XML::Node, @config : Config)
       end
 
       def title
-        document_title || og_title
+        encode(document_title || og_title)
       rescue
         nil
       end
 
       def description
         meta_descriptions = document.xpath_nodes("//meta[@name='description']")
-        unless meta_descriptions.empty?
-          meta_descriptions.first.attributes["content"].text.strip.chomp
-        else
-          secondary_description
-        end
+        description = unless meta_descriptions.empty?
+                        meta_descriptions.first.attributes["content"].text.strip.chomp
+                      else
+                        secondary_description
+                      end
+        encode(description)
       rescue
         nil
       end
@@ -40,6 +41,11 @@ module Metascraper
       rescue
         nil
       end
+
+      def encode(text : String | Nil) : String | Nil
+        return unless text
+        Utils.new(text, config.charset).encodeToUtf8.as(String)
+      end
     end
   end
 end