From 1b9957e773352e3cc26906ff428d5c12f0de5ad3 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Wed, 9 Oct 2024 15:05:55 -0600 Subject: [PATCH 1/4] MONGOID-5411 allow results to be returned as demongoized hashes --- lib/mongoid/contextual/mongo.rb | 61 +++++++++++++++++++++++++++++---- lib/mongoid/criteria.rb | 39 +++++++++++++++++++++ lib/mongoid/findable.rb | 1 + spec/mongoid/criteria_spec.rb | 56 ++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 6 deletions(-) diff --git a/lib/mongoid/contextual/mongo.rb b/lib/mongoid/contextual/mongo.rb index b93fa32d5e..02c88460a7 100644 --- a/lib/mongoid/contextual/mongo.rb +++ b/lib/mongoid/contextual/mongo.rb @@ -880,8 +880,14 @@ def documents_for_iteration # # @param [ Document ] document The document to yield to. def yield_document(document, &block) - doc = document.respond_to?(:_id) ? - document : Factory.from_db(klass, document, criteria) + doc = if document.respond_to?(:_id) + document + elsif criteria.raw_results? + demongoize_hash(klass, document) + else + Factory.from_db(klass, document, criteria) + end + yield(doc) end @@ -979,6 +985,46 @@ def recursive_demongoize(field_name, value, is_translation) demongoize_with_field(field, value, is_translation) end + # Demongoizes (converts from database to Ruby representation) the values + # of the given hash as if it were the raw representation of a document of + # the given klass. + # + # @param [ Document ] klass the Document class that the given hash ought + # to represent + # @param [ Hash | nil ] hash the Hash instance containing the values to + # demongoize. + # + # @return [ Hash | nil ] the demongoized result (nil if the input Hash + # was nil) + # + # @api private + def demongoize_hash(klass, hash) + return nil unless hash + + hash.each_with_object({}) do |(key, value), new_hash| + # does the key represent a declared field on the document? + if (field = klass.fields[key]) + new_hash[key] = field.demongoize(value) + next + end + + # does the key represent an emebedded relation on the document? + aliased_name = klass.aliased_associations[key] || key + if (assoc = klass.relations[aliased_name]) + new_hash[key] = case value + when Array then value.map { |hash| demongoize_hash(assoc.klass, hash) } + when Hash then demongoize_hash(assoc.klass, value) + else value + end + next + end + + # if its not a field or a relation, then we just pass it through + # literally + new_hash[key] = value + end + end + # Demongoize the value for the given field. If the field is nil or the # field is a translations field, the value is demongoized using its class. # @@ -1013,10 +1059,13 @@ def demongoize_with_field(field, value, is_translation) # @return [ Array | Document ] The list of documents or a # single document. def process_raw_docs(raw_docs, limit) - docs = raw_docs.map do |d| - Factory.from_db(klass, d, criteria) - end - docs = eager_load(docs) + docs = if criteria.raw_results? + raw_docs.map { |doc| demongoize_hash(klass, doc) } + else + mapped = raw_docs.map { |doc| Factory.from_db(klass, doc, criteria) } + eager_load(mapped) + end + limit ? docs : docs.first end diff --git a/lib/mongoid/criteria.rb b/lib/mongoid/criteria.rb index ca024cace9..be620242fe 100644 --- a/lib/mongoid/criteria.rb +++ b/lib/mongoid/criteria.rb @@ -172,6 +172,43 @@ def embedded? !!@embedded end + # Produce a clone of the current criteria object with it's "raw" + # setting set to the given value. A criteria set to "raw" will return + # all results as raw, demongoized hashes. When "raw" is not set, the + # criteria will return all results as instantiated Document instances. + # + # @example Return query results as raw hashes: + # Person.where(city: 'Boston').raw + # + # @param [ true | false ] raw_results Whether the new criteria should be + # placed in "raw" mode or not. + # + # @return [ Criteria ] the cloned criteria object. + def raw(raw_results = true) + clone.tap do |criteria| + criteria._raw_results = raw_results + end + end + + # An internal helper for setting the "raw" flag on a given criteria + # object. + # + # @param [ true | false ] raw_results Whether the criteria should be placed + # in "raw" mode or not. + # + # @api private + def _raw_results=(raw_results) + @raw_results = raw_results + end + + # Predicate that answers the question: is this criteria object currently + # in raw mode? (See #raw for a description of raw mode.) + # + # @return [ true | false ] whether the criteria is in raw mode or not. + def raw_results? + @raw_results + end + # Extract a single id from the provided criteria. Could be in an $and # query or a straight _id query. # @@ -278,6 +315,7 @@ def merge!(other) self.documents = other.documents.dup unless other.documents.empty? self.scoping_options = other.scoping_options self.inclusions = (inclusions + other.inclusions).uniq + self._raw_results = self.raw_results? || other.raw_results? self end @@ -513,6 +551,7 @@ def initialize_copy(other) @inclusions = other.inclusions.dup @scoping_options = other.scoping_options @documents = other.documents.dup + @raw_results = other.raw_results? @context = nil super end diff --git a/lib/mongoid/findable.rb b/lib/mongoid/findable.rb index b533d425b1..93a12ce169 100644 --- a/lib/mongoid/findable.rb +++ b/lib/mongoid/findable.rb @@ -46,6 +46,7 @@ module Findable :none, :pick, :pluck, + :raw, :read, :second, :second!, diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index c4ef5ddf39..b5bfdb92a1 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -2269,6 +2269,62 @@ def self.ages; self; end end end + describe '#raw' do + let(:results) { criteria.to_a } + let(:result) { results[0] } + + context 'without associations' do + before do + Band.create(name: 'the band', + active: true, + genres: %w[ abc def ], + member_count: 112, + rating: 4.2, + created: Time.now, + updated: Time.now, + sales: 1_234_567.89, + decimal: 9_876_543.21, + decibels: 140..170, + deleted: false, + mojo: Math::PI, + tags: { 'one' => 1, 'two' => 2 }, + location: LatLng.new(41.74, -111.83)) + end + + let(:criteria) { Band.where(name: 'the band').raw } + + it 'returns a hash' do + expect(result).to be_a(Hash) + end + + it 'demongoizes the result' do + expect(result['genres']).to be_a(Array) + expect(result['decibels']).to be_a(Range) + expect(result['location']).to be_a(LatLng) + end + end + + context 'with associations' do + before do + Person.create({ + addresses: [ Address.new(end_date: 2.months.from_now) ], + passport: Passport.new(exp: 1.year.from_now) + }) + end + + let(:criteria) { Person.raw } + + it 'demongoizes the embedded relation' do + expect(result['addresses']).to be_a(Array) + expect(result['addresses'][0]['end_date']).to be_a(Date) + + # `pass` is how it is stored, `passport` is how it is aliased + expect(result['pass']).to be_a(Hash) + expect(result['pass']['exp']).to be_a(Date) + end + end + end + describe "#max_scan" do max_server_version '4.0' From 333fde49a9ff08f0c306e72f02b7b7898b66248c Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Fri, 11 Oct 2024 08:47:13 -0600 Subject: [PATCH 2/4] tests --- spec/mongoid/contextual/mongo_spec.rb | 16 +++++++++++++--- spec/mongoid/criteria_spec.rb | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/spec/mongoid/contextual/mongo_spec.rb b/spec/mongoid/contextual/mongo_spec.rb index 683d3c2aee..4cead4cb8c 100644 --- a/spec/mongoid/contextual/mongo_spec.rb +++ b/spec/mongoid/contextual/mongo_spec.rb @@ -1240,16 +1240,26 @@ subscriber = Mrss::EventSubscriber.new context.view.client.subscribe(Mongo::Monitoring::COMMAND, subscriber) - enum.next + # first batch + 5.times { enum.next } find_events = subscriber.all_events.select do |evt| evt.command_name == 'find' end - expect(find_events.length).to be(2) + expect(find_events.length).to be > 0 + get_more_events = subscriber.all_events.select do |evt| + evt.command_name == 'getMore' + end + expect(get_more_events.length).to be == 0 + + # force the second batch to be loaded + enum.next + get_more_events = subscriber.all_events.select do |evt| evt.command_name == 'getMore' end - expect(get_more_events.length).to be(0) + expect(get_more_events.length).to be > 0 + ensure context.view.client.unsubscribe(Mongo::Monitoring::COMMAND, subscriber) end diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index b5bfdb92a1..1e5b40546b 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -2323,6 +2323,30 @@ def self.ages; self; end expect(result['pass']['exp']).to be_a(Date) end end + + context 'with projections' do + before { Person.create(title: 'sir', dob: Date.new(1980, 1, 1)) } + + context 'using #only' do + let(:criteria) { Person.only(:dob).raw } + + it 'produces a hash with only the _id and the requested key' do + expect(result).to be_a(Hash) + expect(result.keys).to be == %w[ _id dob ] + expect(result['dob']).to be == Date.new(1980, 1, 1) + end + end + + context 'using #without' do + let(:criteria) { Person.without(:dob).raw } + + it 'produces a hash that excludes requested key' do + expect(result).to be_a(Hash) + expect(result.keys).not_to include('dob') + expect(result.keys).to be_present + end + end + end end describe "#max_scan" do From df098f47780928a990a517d3eb2421735beee0fc Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 14 Oct 2024 14:11:39 -0600 Subject: [PATCH 3/4] modify the hash in-place as an optimization --- lib/mongoid/contextual/mongo.rb | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/mongoid/contextual/mongo.rb b/lib/mongoid/contextual/mongo.rb index 02c88460a7..1e404be269 100644 --- a/lib/mongoid/contextual/mongo.rb +++ b/lib/mongoid/contextual/mongo.rb @@ -989,6 +989,10 @@ def recursive_demongoize(field_name, value, is_translation) # of the given hash as if it were the raw representation of a document of # the given klass. # + # @note this method will modify the given hash, in-place, for performance + # reasons. If you wish to preserve the original hash, duplicate it before + # passing it to this method. + # # @param [ Document ] klass the Document class that the given hash ought # to represent # @param [ Hash | nil ] hash the Hash instance containing the values to @@ -1001,28 +1005,26 @@ def recursive_demongoize(field_name, value, is_translation) def demongoize_hash(klass, hash) return nil unless hash - hash.each_with_object({}) do |(key, value), new_hash| + hash.each_key do |key| + value = hash[key] + # does the key represent a declared field on the document? if (field = klass.fields[key]) - new_hash[key] = field.demongoize(value) + hash[key] = field.demongoize(value) next end # does the key represent an emebedded relation on the document? aliased_name = klass.aliased_associations[key] || key if (assoc = klass.relations[aliased_name]) - new_hash[key] = case value - when Array then value.map { |hash| demongoize_hash(assoc.klass, hash) } - when Hash then demongoize_hash(assoc.klass, value) - else value - end - next + case value + when Array then value.each { |h| demongoize_hash(assoc.klass, h) } + when Hash then demongoize_hash(assoc.klass, value) + end end - - # if its not a field or a relation, then we just pass it through - # literally - new_hash[key] = value end + + hash end # Demongoize the value for the given field. If the field is nil or the From 11553115fff880ff9e7fc41c7f4a38d8052a3e5e Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Thu, 17 Oct 2024 15:33:53 -0600 Subject: [PATCH 4/4] Add a new default mode for raw Returns the hashes exactly as fetched from the database. --- lib/mongoid/contextual/mongo.rb | 12 +- lib/mongoid/criteria.rb | 50 +++++--- spec/mongoid/criteria_spec.rb | 197 +++++++++++++++++++++++--------- 3 files changed, 191 insertions(+), 68 deletions(-) diff --git a/lib/mongoid/contextual/mongo.rb b/lib/mongoid/contextual/mongo.rb index 1e404be269..4972c83356 100644 --- a/lib/mongoid/contextual/mongo.rb +++ b/lib/mongoid/contextual/mongo.rb @@ -883,7 +883,11 @@ def yield_document(document, &block) doc = if document.respond_to?(:_id) document elsif criteria.raw_results? - demongoize_hash(klass, document) + if criteria.typecast_results? + demongoize_hash(klass, document) + else + document + end else Factory.from_db(klass, document, criteria) end @@ -1062,7 +1066,11 @@ def demongoize_with_field(field, value, is_translation) # single document. def process_raw_docs(raw_docs, limit) docs = if criteria.raw_results? - raw_docs.map { |doc| demongoize_hash(klass, doc) } + if criteria.typecast_results? + raw_docs.map { |doc| demongoize_hash(klass, doc) } + else + raw_docs + end else mapped = raw_docs.map { |doc| Factory.from_db(klass, doc, criteria) } eager_load(mapped) diff --git a/lib/mongoid/criteria.rb b/lib/mongoid/criteria.rb index be620242fe..52cafa3204 100644 --- a/lib/mongoid/criteria.rb +++ b/lib/mongoid/criteria.rb @@ -174,39 +174,63 @@ def embedded? # Produce a clone of the current criteria object with it's "raw" # setting set to the given value. A criteria set to "raw" will return - # all results as raw, demongoized hashes. When "raw" is not set, the - # criteria will return all results as instantiated Document instances. + # all results as raw hashes. If `typed` is true, the values in the hashes + # will be typecast according to the fields that they correspond to. + # + # When "raw" is not set (or if `raw_results` is false), the criteria will + # return all results as instantiated Document instances. # # @example Return query results as raw hashes: # Person.where(city: 'Boston').raw # # @param [ true | false ] raw_results Whether the new criteria should be # placed in "raw" mode or not. + # @param [ true | false ] typed Whether the raw results should be typecast + # before being returned. Default is true if raw_results is false, and + # false otherwise. # # @return [ Criteria ] the cloned criteria object. - def raw(raw_results = true) + def raw(raw_results = true, typed: nil) + # default for typed is true when raw_results is false, and false when + # raw_results is true. + typed = !raw_results if typed.nil? + + if !typed && !raw_results + raise ArgumentError, 'instantiated results must be typecast' + end + clone.tap do |criteria| - criteria._raw_results = raw_results + criteria._raw_results = { raw: raw_results, typed: typed } end end - # An internal helper for setting the "raw" flag on a given criteria + # An internal helper for getting/setting the "raw" flag on a given criteria # object. # - # @param [ true | false ] raw_results Whether the criteria should be placed - # in "raw" mode or not. + # @return [ nil | Hash ] If set, it is a hash with two keys, :raw and :typed, + # that describe whether raw results should be returned, and whether they + # ought to be typecast. # # @api private - def _raw_results=(raw_results) - @raw_results = raw_results - end + attr_accessor :_raw_results # Predicate that answers the question: is this criteria object currently # in raw mode? (See #raw for a description of raw mode.) # # @return [ true | false ] whether the criteria is in raw mode or not. def raw_results? - @raw_results + _raw_results && _raw_results[:raw] + end + + # Predicate that answers the question: should the results returned by + # this criteria object be typecast? (See #raw for a description of this.) + # The answer is meaningless unless #raw_results? is true, since if + # instantiated document objects are returned they will always be typecast. + # + # @return [ true | false ] whether the criteria should return typecast + # results. + def typecast_results? + _raw_results && _raw_results[:typed] end # Extract a single id from the provided criteria. Could be in an $and @@ -315,7 +339,7 @@ def merge!(other) self.documents = other.documents.dup unless other.documents.empty? self.scoping_options = other.scoping_options self.inclusions = (inclusions + other.inclusions).uniq - self._raw_results = self.raw_results? || other.raw_results? + self._raw_results = self._raw_results || other._raw_results self end @@ -551,7 +575,7 @@ def initialize_copy(other) @inclusions = other.inclusions.dup @scoping_options = other.scoping_options @documents = other.documents.dup - @raw_results = other.raw_results? + self._raw_results = other._raw_results @context = nil super end diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index 1e5b40546b..24832248bc 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -2270,80 +2270,171 @@ def self.ages; self; end end describe '#raw' do - let(:results) { criteria.to_a } let(:result) { results[0] } - context 'without associations' do - before do - Band.create(name: 'the band', - active: true, - genres: %w[ abc def ], - member_count: 112, - rating: 4.2, - created: Time.now, - updated: Time.now, - sales: 1_234_567.89, - decimal: 9_876_543.21, - decibels: 140..170, - deleted: false, - mojo: Math::PI, - tags: { 'one' => 1, 'two' => 2 }, - location: LatLng.new(41.74, -111.83)) + context 'when the parameters are inconsistent' do + let(:results) { criteria.raw(false, typed: false).to_a } + let(:criteria) { Person } + + it 'raises an ArgumentError' do + expect { result }.to raise_error(ArgumentError) end + end - let(:criteria) { Band.where(name: 'the band').raw } + context 'when returning untyped results' do + let(:results) { criteria.raw.to_a } - it 'returns a hash' do - expect(result).to be_a(Hash) - end + context 'without associations' do + before do + Band.create(name: 'the band', + active: true, + genres: %w[ abc def ], + member_count: 112, + rating: 4.2, + created: Time.now, + updated: Time.now, + sales: 1_234_567.89, + decimal: 9_876_543.21, + decibels: 140..170, + deleted: false, + mojo: Math::PI, + tags: { 'one' => 1, 'two' => 2 }, + location: LatLng.new(41.74, -111.83)) + end + + let(:criteria) { Band.where(name: 'the band') } + + it 'returns a hash' do + expect(result).to be_a(Hash) + end - it 'demongoizes the result' do - expect(result['genres']).to be_a(Array) - expect(result['decibels']).to be_a(Range) - expect(result['location']).to be_a(LatLng) + it 'does not demongoize the result' do + expect(result['genres']).to be_a(Array) + expect(result['decibels']).to be == { 'min' => 140, 'max' => 170 } + expect(result['location']).to be == [ -111.83, 41.74 ] + end end - end - context 'with associations' do - before do - Person.create({ - addresses: [ Address.new(end_date: 2.months.from_now) ], - passport: Passport.new(exp: 1.year.from_now) - }) + context 'with associations' do + before do + Person.create({ + addresses: [ Address.new(end_date: 2.months.from_now) ], + passport: Passport.new(exp: 1.year.from_now) + }) + end + + let(:criteria) { Person } + + it 'demongoizes the embedded relation' do + expect(result['addresses']).to be_a(Array) + expect(result['addresses'][0]['end_date']).to be_a(Time) + + # `pass` is how it is stored, `passport` is how it is aliased + expect(result['pass']).to be_a(Hash) + expect(result['pass']['exp']).to be_a(Time) + end end - let(:criteria) { Person.raw } + context 'with projections' do + before { Person.create(title: 'sir', dob: Date.new(1980, 1, 1)) } - it 'demongoizes the embedded relation' do - expect(result['addresses']).to be_a(Array) - expect(result['addresses'][0]['end_date']).to be_a(Date) + context 'using #only' do + let(:criteria) { Person.only(:dob) } - # `pass` is how it is stored, `passport` is how it is aliased - expect(result['pass']).to be_a(Hash) - expect(result['pass']['exp']).to be_a(Date) + it 'produces a hash with only the _id and the requested key' do + expect(result).to be_a(Hash) + expect(result.keys).to be == %w[ _id dob ] + expect(result['dob']).to be == Date.new(1980, 1, 1) + end + end + + context 'using #without' do + let(:criteria) { Person.without(:dob) } + + it 'produces a hash that excludes requested key' do + expect(result).to be_a(Hash) + expect(result.keys).not_to include('dob') + expect(result.keys).to be_present + end + end end end - context 'with projections' do - before { Person.create(title: 'sir', dob: Date.new(1980, 1, 1)) } - - context 'using #only' do - let(:criteria) { Person.only(:dob).raw } + context 'when returning typed results' do + let(:results) { criteria.raw(typed: true).to_a } - it 'produces a hash with only the _id and the requested key' do + context 'without associations' do + before do + Band.create(name: 'the band', + active: true, + genres: %w[ abc def ], + member_count: 112, + rating: 4.2, + created: Time.now, + updated: Time.now, + sales: 1_234_567.89, + decimal: 9_876_543.21, + decibels: 140..170, + deleted: false, + mojo: Math::PI, + tags: { 'one' => 1, 'two' => 2 }, + location: LatLng.new(41.74, -111.83)) + end + + let(:criteria) { Band.where(name: 'the band') } + + it 'returns a hash' do expect(result).to be_a(Hash) - expect(result.keys).to be == %w[ _id dob ] - expect(result['dob']).to be == Date.new(1980, 1, 1) + end + + it 'demongoizes the result' do + expect(result['genres']).to be_a(Array) + expect(result['decibels']).to be_a(Range) + expect(result['location']).to be_a(LatLng) end end - context 'using #without' do - let(:criteria) { Person.without(:dob).raw } + context 'with associations' do + before do + Person.create({ + addresses: [ Address.new(end_date: 2.months.from_now) ], + passport: Passport.new(exp: 1.year.from_now) + }) + end - it 'produces a hash that excludes requested key' do - expect(result).to be_a(Hash) - expect(result.keys).not_to include('dob') - expect(result.keys).to be_present + let(:criteria) { Person } + + it 'demongoizes the embedded relation' do + expect(result['addresses']).to be_a(Array) + expect(result['addresses'][0]['end_date']).to be_a(Date) + + # `pass` is how it is stored, `passport` is how it is aliased + expect(result['pass']).to be_a(Hash) + expect(result['pass']['exp']).to be_a(Date) + end + end + + context 'with projections' do + before { Person.create(title: 'sir', dob: Date.new(1980, 1, 1)) } + + context 'using #only' do + let(:criteria) { Person.only(:dob) } + + it 'produces a hash with only the _id and the requested key' do + expect(result).to be_a(Hash) + expect(result.keys).to be == %w[ _id dob ] + expect(result['dob']).to be == Date.new(1980, 1, 1) + end + end + + context 'using #without' do + let(:criteria) { Person.without(:dob) } + + it 'produces a hash that excludes requested key' do + expect(result).to be_a(Hash) + expect(result.keys).not_to include('dob') + expect(result.keys).to be_present + end end end end