Elasticsearch: Highlighting hits from within attachment

elasticsearch, ruby-on-rails, syntax-highlighting, tire

Solution

I figured it out! Finally...

problem was with my mapping syntax in Article class. Needed to rename ":file" to ":attachment".

  tire.mapping do
    indexes :id, :type =>'integer'
    indexes :title
    indexes :content
    indexes :published_on, :type => 'date'
    indexes :attachment, :type => 'attachment', #:null_value => 'missing_file',
                            :fields => {
                            :name       => { :store => 'yes' },  # exists?!?
                            :content    => { :store => 'yes' },
                            :title      => { :store => 'yes' },
  # WRONG! see next line => :file       => { :term_vector => 'with_positions_offsets', :store => 'yes' },
                            :attachment => { :term_vector => 'with_positions_offsets', :store => 'yes' },
                            :date       => { :store => 'yes' }
                          }

Problem

I'm having trouble getting the highlighting to work with Elasticsearch (and Tire) in a Rails app. I can successfully index PDF attachments and query them but I cannot get the highlighting to work. Not that familiar with ES so not sure where to look to troubleshoot. Will start with mappings and a curl query but feel free to ask for more info. ``` class Article < ActiveRecord::Base include Tire::Model::Search include Tire::Model::Callbacks attr_accessible :title, :content, :published_on, :filename mapping do indexes :id, :type =>'integer' indexes :title indexes :content indexes :published_on, :type => 'date' indexes :attachment, :type => 'attachment', :fields => { :name => { :store => 'yes' }, :content => { :store => 'yes' }, :title => { :store => 'yes' }, :file => { :term_vector => 'with_positions_offsets', :store => 'yes' }, :date => { :store => 'yes' } } end def to_indexed_json to_json(:methods => [:attachment]) end def attachment if filename.present? path_to_pdf = "/Volumes/Calvin/sample_pdfs/#{filename}.pdf" Base64.encode64(open(path_to_pdf) { |pdf| pdf.read }) else Base64.encode64("missing") end end end ``` Mappings (via Curl): ``` $ curl -XGET 'http://localhost:9200/_mapping?pretty=true' { "articles" : { "article" : { "properties" : { "attachment" : { "type" : "attachment", "path" : "full", "fields" : { "attachment" : { "type" : "string" }, "title" : { "type" : "string", "store" : "yes" }, "name" : { "type" : "string", "store" : "yes" }, "date" : { "type" : "date", "ignore_malformed" : false, "store" : "yes", "format" : "dateOptionalTime" }, "content_type" : { "type" : "string" } } }, "content" : { "type" : "string" }, "created_at" : { "type" : "date", "ignore_malformed" : false, "format" : "dateOptionalTime" }, "filename" : { "type" : "string" }, "id" : { "type" : "integer", "ignore_malformed" : false }, "published_on" : { "type" : "date", "ignore_malformed" : false, "format" : "dateOptionalTime" }, "title" : { "type" : "string" }, "updated_at" : { "type" : "date", "ignore_malformed" : false, "format" : "dateOptionalTime" } } } } }% ``` A query with a 'hit' inside a 125 page indexed PDF: ``` $ curl "localhost:9200/_search?pretty=true" -d '{ quote> "fields" : ["title"], quote> "query" : { quote> "query_string" : { quote> "query" : "xerox" quote> } quote> }, quote> "highlight" : { quote> "fields" : { quote> "attachment" : {} quote> } quote> } quote> }' { "took" : 1077, "timed_out" : false, "_shards" : { "total" : 5, "successful" : 5, "failed" : 0 }, "hits" : { "total" : 1, "max_score" : 0.036417194, "hits" : [ { "_index" : "articles", "_type" : "article", "_id" : "13", "_score" : 0.036417194, "fields" : { "title" : "F-E12" } } ] } }% ``` I was expecting a section like: ``` "highlight" : { "attachment" : [ "\nLast Year <em>Xerox</em> moved their facilities" ] } ``` Thanks for any help! Edit2: adjusted query (changed `attachment` to `attachment.file`) to no avail: ``` $ curl "localhost:9200/_search?pretty=true" -d '{ "fields" : ["title","attachment"], "query" : {"query_string" : {"query" : "xerox"}}, "highlight" : {"fields" : {"attachment.file" : {}}} }' { "took" : 221, "timed_out" : false, "_shards" : { "total" : 5, "successful" : 5, "failed" : 0 }, "hits" : { "total" : 1, "max_score" : 0.036417194, "hits" : [ { "_index" : "articles", "_type" : "article", "_id" : "13", "_score" : 0.036417194, "fields" : { "title" : "F-E12", "attachment" : "JVBERi0xLjYNJeLjz9MNCjk4NSAwIG9iag08PC9MaW5lYXJpemVkIDEvTCA...\n" } } ] } } ``` Edit3 (remove "fields"): ``` $ curl "localhost:9200/_search?pretty=true" -d '{ > "query" : {"query_string" : {"query" : "xerox"}}, > "highlight" : {"fields" : {"attachment" : {}}} > }' { "took" : 1078, "timed_out" : false, "_shards" : { "total" : 5, "successful" : 5, "failed" : 0 }, "hits" : { "total" : 1, "max_score" : 0.036417194, "hits" : [ { "_index" : "articles", "_type" : "article", "_id" : "13", "_score" : 0.036417194, "_source" : {"content":"Real report","created_at":"2012-08-28T22:44:08Z","filename":"F-E12","id":13,"published_on":"2007-12-28","title":"F-E12","updated_at":"2012-08-28T22:44:08Z","attachment":"JVBERi0xLjYNJeLjz9MNCjk4NSAwIG9iag08PC9MaW5lYXJpemVkID...\n" } } ] } } ``` Edit4 (mapping from Attachment Type in Action tutorial): ``` $ curl -XGET 'http://localhost:9200/test/_mapping?pretty=true' { "test" : { "attachment" : { "properties" : { "file" : { "type" : "attachment", "path" : "full", "fields" : { "file" : { #<== This appears to be missing "type" : "string", #<== from my Articles mapping "store" : "yes", #<== "term_vector" : "with_positions_offsets" #<== }, "author" : { "type" : "string" }, "title" : { "type" : "string", "store" : "yes" }, "name" : { "type" : "string" }, "date" : { "type" : "date", "ignore_malformed" : false, "format" : "dateOptionalTime" }, "keywords" : { "type" : "string" }, "content_type" : { "type" : "string" } } } } } } } ```

Original source