diff options
Diffstat (limited to 'parts/django/django/db/models/sql')
| -rw-r--r-- | parts/django/django/db/models/sql/__init__.py | 7 | ||||
| -rw-r--r-- | parts/django/django/db/models/sql/aggregates.py | 128 | ||||
| -rw-r--r-- | parts/django/django/db/models/sql/compiler.py | 979 | ||||
| -rw-r--r-- | parts/django/django/db/models/sql/constants.py | 37 | ||||
| -rw-r--r-- | parts/django/django/db/models/sql/datastructures.py | 46 | ||||
| -rw-r--r-- | parts/django/django/db/models/sql/expressions.py | 87 | ||||
| -rw-r--r-- | parts/django/django/db/models/sql/query.py | 1860 | ||||
| -rw-r--r-- | parts/django/django/db/models/sql/subqueries.py | 212 | ||||
| -rw-r--r-- | parts/django/django/db/models/sql/where.py | 345 |
9 files changed, 0 insertions, 3701 deletions
diff --git a/parts/django/django/db/models/sql/__init__.py b/parts/django/django/db/models/sql/__init__.py deleted file mode 100644 index 7310982..0000000 --- a/parts/django/django/db/models/sql/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from query import * -from subqueries import * -from where import AND, OR -from datastructures import EmptyResultSet - -__all__ = ['Query', 'AND', 'OR', 'EmptyResultSet'] - diff --git a/parts/django/django/db/models/sql/aggregates.py b/parts/django/django/db/models/sql/aggregates.py deleted file mode 100644 index 207bc0c..0000000 --- a/parts/django/django/db/models/sql/aggregates.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -Classes to represent the default SQL aggregate functions -""" - -class AggregateField(object): - """An internal field mockup used to identify aggregates in the - data-conversion parts of the database backend. - """ - def __init__(self, internal_type): - self.internal_type = internal_type - - def get_internal_type(self): - return self.internal_type - -ordinal_aggregate_field = AggregateField('IntegerField') -computed_aggregate_field = AggregateField('FloatField') - -class Aggregate(object): - """ - Default SQL Aggregate. - """ - is_ordinal = False - is_computed = False - sql_template = '%(function)s(%(field)s)' - - def __init__(self, col, source=None, is_summary=False, **extra): - """Instantiate an SQL aggregate - - * col is a column reference describing the subject field - of the aggregate. It can be an alias, or a tuple describing - a table and column name. - * source is the underlying field or aggregate definition for - the column reference. If the aggregate is not an ordinal or - computed type, this reference is used to determine the coerced - output type of the aggregate. - * extra is a dictionary of additional data to provide for the - aggregate definition - - Also utilizes the class variables: - * sql_function, the name of the SQL function that implements the - aggregate. - * sql_template, a template string that is used to render the - aggregate into SQL. - * is_ordinal, a boolean indicating if the output of this aggregate - is an integer (e.g., a count) - * is_computed, a boolean indicating if this output of this aggregate - is a computed float (e.g., an average), regardless of the input - type. - - """ - self.col = col - self.source = source - self.is_summary = is_summary - self.extra = extra - - # Follow the chain of aggregate sources back until you find an - # actual field, or an aggregate that forces a particular output - # type. This type of this field will be used to coerce values - # retrieved from the database. - tmp = self - - while tmp and isinstance(tmp, Aggregate): - if getattr(tmp, 'is_ordinal', False): - tmp = ordinal_aggregate_field - elif getattr(tmp, 'is_computed', False): - tmp = computed_aggregate_field - else: - tmp = tmp.source - - self.field = tmp - - def relabel_aliases(self, change_map): - if isinstance(self.col, (list, tuple)): - self.col = (change_map.get(self.col[0], self.col[0]), self.col[1]) - - def as_sql(self, qn, connection): - "Return the aggregate, rendered as SQL." - - if hasattr(self.col, 'as_sql'): - field_name = self.col.as_sql(qn, connection) - elif isinstance(self.col, (list, tuple)): - field_name = '.'.join([qn(c) for c in self.col]) - else: - field_name = self.col - - params = { - 'function': self.sql_function, - 'field': field_name - } - params.update(self.extra) - - return self.sql_template % params - - -class Avg(Aggregate): - is_computed = True - sql_function = 'AVG' - -class Count(Aggregate): - is_ordinal = True - sql_function = 'COUNT' - sql_template = '%(function)s(%(distinct)s%(field)s)' - - def __init__(self, col, distinct=False, **extra): - super(Count, self).__init__(col, distinct=distinct and 'DISTINCT ' or '', **extra) - -class Max(Aggregate): - sql_function = 'MAX' - -class Min(Aggregate): - sql_function = 'MIN' - -class StdDev(Aggregate): - is_computed = True - - def __init__(self, col, sample=False, **extra): - super(StdDev, self).__init__(col, **extra) - self.sql_function = sample and 'STDDEV_SAMP' or 'STDDEV_POP' - -class Sum(Aggregate): - sql_function = 'SUM' - -class Variance(Aggregate): - is_computed = True - - def __init__(self, col, sample=False, **extra): - super(Variance, self).__init__(col, **extra) - self.sql_function = sample and 'VAR_SAMP' or 'VAR_POP' diff --git a/parts/django/django/db/models/sql/compiler.py b/parts/django/django/db/models/sql/compiler.py deleted file mode 100644 index d464061..0000000 --- a/parts/django/django/db/models/sql/compiler.py +++ /dev/null @@ -1,979 +0,0 @@ -from django.core.exceptions import FieldError -from django.db import connections -from django.db.backends.util import truncate_name -from django.db.models.sql.constants import * -from django.db.models.sql.datastructures import EmptyResultSet -from django.db.models.sql.expressions import SQLEvaluator -from django.db.models.sql.query import get_proxied_model, get_order_dir, \ - select_related_descend, Query - -class SQLCompiler(object): - def __init__(self, query, connection, using): - self.query = query - self.connection = connection - self.using = using - self.quote_cache = {} - - def pre_sql_setup(self): - """ - Does any necessary class setup immediately prior to producing SQL. This - is for things that can't necessarily be done in __init__ because we - might not have all the pieces in place at that time. - """ - if not self.query.tables: - self.query.join((None, self.query.model._meta.db_table, None, None)) - if (not self.query.select and self.query.default_cols and not - self.query.included_inherited_models): - self.query.setup_inherited_models() - if self.query.select_related and not self.query.related_select_cols: - self.fill_related_selections() - - def quote_name_unless_alias(self, name): - """ - A wrapper around connection.ops.quote_name that doesn't quote aliases - for table names. This avoids problems with some SQL dialects that treat - quoted strings specially (e.g. PostgreSQL). - """ - if name in self.quote_cache: - return self.quote_cache[name] - if ((name in self.query.alias_map and name not in self.query.table_map) or - name in self.query.extra_select): - self.quote_cache[name] = name - return name - r = self.connection.ops.quote_name(name) - self.quote_cache[name] = r - return r - - def as_sql(self, with_limits=True, with_col_aliases=False): - """ - Creates the SQL for this query. Returns the SQL string and list of - parameters. - - If 'with_limits' is False, any limit/offset information is not included - in the query. - """ - self.pre_sql_setup() - out_cols = self.get_columns(with_col_aliases) - ordering, ordering_group_by = self.get_ordering() - - # This must come after 'select' and 'ordering' -- see docstring of - # get_from_clause() for details. - from_, f_params = self.get_from_clause() - - qn = self.quote_name_unless_alias - - where, w_params = self.query.where.as_sql(qn=qn, connection=self.connection) - having, h_params = self.query.having.as_sql(qn=qn, connection=self.connection) - params = [] - for val in self.query.extra_select.itervalues(): - params.extend(val[1]) - - result = ['SELECT'] - if self.query.distinct: - result.append('DISTINCT') - result.append(', '.join(out_cols + self.query.ordering_aliases)) - - result.append('FROM') - result.extend(from_) - params.extend(f_params) - - if where: - result.append('WHERE %s' % where) - params.extend(w_params) - - grouping, gb_params = self.get_grouping() - if grouping: - if ordering: - # If the backend can't group by PK (i.e., any database - # other than MySQL), then any fields mentioned in the - # ordering clause needs to be in the group by clause. - if not self.connection.features.allows_group_by_pk: - for col, col_params in ordering_group_by: - if col not in grouping: - grouping.append(str(col)) - gb_params.extend(col_params) - else: - ordering = self.connection.ops.force_no_ordering() - result.append('GROUP BY %s' % ', '.join(grouping)) - params.extend(gb_params) - - if having: - result.append('HAVING %s' % having) - params.extend(h_params) - - if ordering: - result.append('ORDER BY %s' % ', '.join(ordering)) - - if with_limits: - if self.query.high_mark is not None: - result.append('LIMIT %d' % (self.query.high_mark - self.query.low_mark)) - if self.query.low_mark: - if self.query.high_mark is None: - val = self.connection.ops.no_limit_value() - if val: - result.append('LIMIT %d' % val) - result.append('OFFSET %d' % self.query.low_mark) - - return ' '.join(result), tuple(params) - - def as_nested_sql(self): - """ - Perform the same functionality as the as_sql() method, returning an - SQL string and parameters. However, the alias prefixes are bumped - beforehand (in a copy -- the current query isn't changed), and any - ordering is removed if the query is unsliced. - - Used when nesting this query inside another. - """ - obj = self.query.clone() - if obj.low_mark == 0 and obj.high_mark is None: - # If there is no slicing in use, then we can safely drop all ordering - obj.clear_ordering(True) - obj.bump_prefix() - return obj.get_compiler(connection=self.connection).as_sql() - - def get_columns(self, with_aliases=False): - """ - Returns the list of columns to use in the select statement. If no - columns have been specified, returns all columns relating to fields in - the model. - - If 'with_aliases' is true, any column names that are duplicated - (without the table names) are given unique aliases. This is needed in - some cases to avoid ambiguity with nested queries. - """ - qn = self.quote_name_unless_alias - qn2 = self.connection.ops.quote_name - result = ['(%s) AS %s' % (col[0], qn2(alias)) for alias, col in self.query.extra_select.iteritems()] - aliases = set(self.query.extra_select.keys()) - if with_aliases: - col_aliases = aliases.copy() - else: - col_aliases = set() - if self.query.select: - only_load = self.deferred_to_columns() - for col in self.query.select: - if isinstance(col, (list, tuple)): - alias, column = col - table = self.query.alias_map[alias][TABLE_NAME] - if table in only_load and col not in only_load[table]: - continue - r = '%s.%s' % (qn(alias), qn(column)) - if with_aliases: - if col[1] in col_aliases: - c_alias = 'Col%d' % len(col_aliases) - result.append('%s AS %s' % (r, c_alias)) - aliases.add(c_alias) - col_aliases.add(c_alias) - else: - result.append('%s AS %s' % (r, qn2(col[1]))) - aliases.add(r) - col_aliases.add(col[1]) - else: - result.append(r) - aliases.add(r) - col_aliases.add(col[1]) - else: - result.append(col.as_sql(qn, self.connection)) - - if hasattr(col, 'alias'): - aliases.add(col.alias) - col_aliases.add(col.alias) - - elif self.query.default_cols: - cols, new_aliases = self.get_default_columns(with_aliases, - col_aliases) - result.extend(cols) - aliases.update(new_aliases) - - max_name_length = self.connection.ops.max_name_length() - result.extend([ - '%s%s' % ( - aggregate.as_sql(qn, self.connection), - alias is not None - and ' AS %s' % qn(truncate_name(alias, max_name_length)) - or '' - ) - for alias, aggregate in self.query.aggregate_select.items() - ]) - - for table, col in self.query.related_select_cols: - r = '%s.%s' % (qn(table), qn(col)) - if with_aliases and col in col_aliases: - c_alias = 'Col%d' % len(col_aliases) - result.append('%s AS %s' % (r, c_alias)) - aliases.add(c_alias) - col_aliases.add(c_alias) - else: - result.append(r) - aliases.add(r) - col_aliases.add(col) - - self._select_aliases = aliases - return result - - def get_default_columns(self, with_aliases=False, col_aliases=None, - start_alias=None, opts=None, as_pairs=False, local_only=False): - """ - Computes the default columns for selecting every field in the base - model. Will sometimes be called to pull in related models (e.g. via - select_related), in which case "opts" and "start_alias" will be given - to provide a starting point for the traversal. - - Returns a list of strings, quoted appropriately for use in SQL - directly, as well as a set of aliases used in the select statement (if - 'as_pairs' is True, returns a list of (alias, col_name) pairs instead - of strings as the first component and None as the second component). - """ - result = [] - if opts is None: - opts = self.query.model._meta - qn = self.quote_name_unless_alias - qn2 = self.connection.ops.quote_name - aliases = set() - only_load = self.deferred_to_columns() - # Skip all proxy to the root proxied model - proxied_model = get_proxied_model(opts) - - if start_alias: - seen = {None: start_alias} - for field, model in opts.get_fields_with_model(): - if local_only and model is not None: - continue - if start_alias: - try: - alias = seen[model] - except KeyError: - if model is proxied_model: - alias = start_alias - else: - link_field = opts.get_ancestor_link(model) - alias = self.query.join((start_alias, model._meta.db_table, - link_field.column, model._meta.pk.column)) - seen[model] = alias - else: - # If we're starting from the base model of the queryset, the - # aliases will have already been set up in pre_sql_setup(), so - # we can save time here. - alias = self.query.included_inherited_models[model] - table = self.query.alias_map[alias][TABLE_NAME] - if table in only_load and field.column not in only_load[table]: - continue - if as_pairs: - result.append((alias, field.column)) - aliases.add(alias) - continue - if with_aliases and field.column in col_aliases: - c_alias = 'Col%d' % len(col_aliases) - result.append('%s.%s AS %s' % (qn(alias), - qn2(field.column), c_alias)) - col_aliases.add(c_alias) - aliases.add(c_alias) - else: - r = '%s.%s' % (qn(alias), qn2(field.column)) - result.append(r) - aliases.add(r) - if with_aliases: - col_aliases.add(field.column) - return result, aliases - - def get_ordering(self): - """ - Returns a tuple containing a list representing the SQL elements in the - "order by" clause, and the list of SQL elements that need to be added - to the GROUP BY clause as a result of the ordering. - - Also sets the ordering_aliases attribute on this instance to a list of - extra aliases needed in the select. - - Determining the ordering SQL can change the tables we need to include, - so this should be run *before* get_from_clause(). - """ - if self.query.extra_order_by: - ordering = self.query.extra_order_by - elif not self.query.default_ordering: - ordering = self.query.order_by - else: - ordering = self.query.order_by or self.query.model._meta.ordering - qn = self.quote_name_unless_alias - qn2 = self.connection.ops.quote_name - distinct = self.query.distinct - select_aliases = self._select_aliases - result = [] - group_by = [] - ordering_aliases = [] - if self.query.standard_ordering: - asc, desc = ORDER_DIR['ASC'] - else: - asc, desc = ORDER_DIR['DESC'] - - # It's possible, due to model inheritance, that normal usage might try - # to include the same field more than once in the ordering. We track - # the table/column pairs we use and discard any after the first use. - processed_pairs = set() - - for field in ordering: - if field == '?': - result.append(self.connection.ops.random_function_sql()) - continue - if isinstance(field, int): - if field < 0: - order = desc - field = -field - else: - order = asc - result.append('%s %s' % (field, order)) - group_by.append((field, [])) - continue - col, order = get_order_dir(field, asc) - if col in self.query.aggregate_select: - result.append('%s %s' % (col, order)) - continue - if '.' in field: - # This came in through an extra(order_by=...) addition. Pass it - # on verbatim. - table, col = col.split('.', 1) - if (table, col) not in processed_pairs: - elt = '%s.%s' % (qn(table), col) - processed_pairs.add((table, col)) - if not distinct or elt in select_aliases: - result.append('%s %s' % (elt, order)) - group_by.append((elt, [])) - elif get_order_dir(field)[0] not in self.query.extra_select: - # 'col' is of the form 'field' or 'field1__field2' or - # '-field1__field2__field', etc. - for table, col, order in self.find_ordering_name(field, - self.query.model._meta, default_order=asc): - if (table, col) not in processed_pairs: - elt = '%s.%s' % (qn(table), qn2(col)) - processed_pairs.add((table, col)) - if distinct and elt not in select_aliases: - ordering_aliases.append(elt) - result.append('%s %s' % (elt, order)) - group_by.append((elt, [])) - else: - elt = qn2(col) - if distinct and col not in select_aliases: - ordering_aliases.append(elt) - result.append('%s %s' % (elt, order)) - group_by.append(self.query.extra_select[col]) - self.query.ordering_aliases = ordering_aliases - return result, group_by - - def find_ordering_name(self, name, opts, alias=None, default_order='ASC', - already_seen=None): - """ - Returns the table alias (the name might be ambiguous, the alias will - not be) and column name for ordering by the given 'name' parameter. - The 'name' is of the form 'field1__field2__...__fieldN'. - """ - name, order = get_order_dir(name, default_order) - pieces = name.split(LOOKUP_SEP) - if not alias: - alias = self.query.get_initial_alias() - field, target, opts, joins, last, extra = self.query.setup_joins(pieces, - opts, alias, False) - alias = joins[-1] - col = target.column - if not field.rel: - # To avoid inadvertent trimming of a necessary alias, use the - # refcount to show that we are referencing a non-relation field on - # the model. - self.query.ref_alias(alias) - - # Must use left outer joins for nullable fields and their relations. - self.query.promote_alias_chain(joins, - self.query.alias_map[joins[0]][JOIN_TYPE] == self.query.LOUTER) - - # If we get to this point and the field is a relation to another model, - # append the default ordering for that model. - if field.rel and len(joins) > 1 and opts.ordering: - # Firstly, avoid infinite loops. - if not already_seen: - already_seen = set() - join_tuple = tuple([self.query.alias_map[j][TABLE_NAME] for j in joins]) - if join_tuple in already_seen: - raise FieldError('Infinite loop caused by ordering.') - already_seen.add(join_tuple) - - results = [] - for item in opts.ordering: - results.extend(self.find_ordering_name(item, opts, alias, - order, already_seen)) - return results - - if alias: - # We have to do the same "final join" optimisation as in - # add_filter, since the final column might not otherwise be part of - # the select set (so we can't order on it). - while 1: - join = self.query.alias_map[alias] - if col != join[RHS_JOIN_COL]: - break - self.query.unref_alias(alias) - alias = join[LHS_ALIAS] - col = join[LHS_JOIN_COL] - return [(alias, col, order)] - - def get_from_clause(self): - """ - Returns a list of strings that are joined together to go after the - "FROM" part of the query, as well as a list any extra parameters that - need to be included. Sub-classes, can override this to create a - from-clause via a "select". - - This should only be called after any SQL construction methods that - might change the tables we need. This means the select columns and - ordering must be done first. - """ - result = [] - qn = self.quote_name_unless_alias - qn2 = self.connection.ops.quote_name - first = True - for alias in self.query.tables: - if not self.query.alias_refcount[alias]: - continue - try: - name, alias, join_type, lhs, lhs_col, col, nullable = self.query.alias_map[alias] - except KeyError: - # Extra tables can end up in self.tables, but not in the - # alias_map if they aren't in a join. That's OK. We skip them. - continue - alias_str = (alias != name and ' %s' % alias or '') - if join_type and not first: - result.append('%s %s%s ON (%s.%s = %s.%s)' - % (join_type, qn(name), alias_str, qn(lhs), - qn2(lhs_col), qn(alias), qn2(col))) - else: - connector = not first and ', ' or '' - result.append('%s%s%s' % (connector, qn(name), alias_str)) - first = False - for t in self.query.extra_tables: - alias, unused = self.query.table_alias(t) - # Only add the alias if it's not already present (the table_alias() - # calls increments the refcount, so an alias refcount of one means - # this is the only reference. - if alias not in self.query.alias_map or self.query.alias_refcount[alias] == 1: - connector = not first and ', ' or '' - result.append('%s%s' % (connector, qn(alias))) - first = False - return result, [] - - def get_grouping(self): - """ - Returns a tuple representing the SQL elements in the "group by" clause. - """ - qn = self.quote_name_unless_alias - result, params = [], [] - if self.query.group_by is not None: - if (len(self.query.model._meta.fields) == len(self.query.select) and - self.connection.features.allows_group_by_pk): - self.query.group_by = [ - (self.query.model._meta.db_table, self.query.model._meta.pk.column) - ] - - group_by = self.query.group_by or [] - - extra_selects = [] - for extra_select, extra_params in self.query.extra_select.itervalues(): - extra_selects.append(extra_select) - params.extend(extra_params) - cols = (group_by + self.query.select + - self.query.related_select_cols + extra_selects) - for col in cols: - if isinstance(col, (list, tuple)): - result.append('%s.%s' % (qn(col[0]), qn(col[1]))) - elif hasattr(col, 'as_sql'): - result.append(col.as_sql(qn, self.connection)) - else: - result.append('(%s)' % str(col)) - return result, params - - def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, - used=None, requested=None, restricted=None, nullable=None, - dupe_set=None, avoid_set=None): - """ - Fill in the information needed for a select_related query. The current - depth is measured as the number of connections away from the root model - (for example, cur_depth=1 means we are looking at models with direct - connections to the root model). - """ - if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: - # We've recursed far enough; bail out. - return - - if not opts: - opts = self.query.get_meta() - root_alias = self.query.get_initial_alias() - self.query.related_select_cols = [] - self.query.related_select_fields = [] - if not used: - used = set() - if dupe_set is None: - dupe_set = set() - if avoid_set is None: - avoid_set = set() - orig_dupe_set = dupe_set - - # Setup for the case when only particular related fields should be - # included in the related selection. - if requested is None: - if isinstance(self.query.select_related, dict): - requested = self.query.select_related - restricted = True - else: - restricted = False - - for f, model in opts.get_fields_with_model(): - if not select_related_descend(f, restricted, requested): - continue - # The "avoid" set is aliases we want to avoid just for this - # particular branch of the recursion. They aren't permanently - # forbidden from reuse in the related selection tables (which is - # what "used" specifies). - avoid = avoid_set.copy() - dupe_set = orig_dupe_set.copy() - table = f.rel.to._meta.db_table - promote = nullable or f.null - if model: - int_opts = opts - alias = root_alias - alias_chain = [] - for int_model in opts.get_base_chain(model): - # Proxy model have elements in base chain - # with no parents, assign the new options - # object and skip to the next base in that - # case - if not int_opts.parents[int_model]: - int_opts = int_model._meta - continue - lhs_col = int_opts.parents[int_model].column - dedupe = lhs_col in opts.duplicate_targets - if dedupe: - avoid.update(self.query.dupe_avoidance.get((id(opts), lhs_col), - ())) - dupe_set.add((opts, lhs_col)) - int_opts = int_model._meta - alias = self.query.join((alias, int_opts.db_table, lhs_col, - int_opts.pk.column), exclusions=used, - promote=promote) - alias_chain.append(alias) - for (dupe_opts, dupe_col) in dupe_set: - self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) - if self.query.alias_map[root_alias][JOIN_TYPE] == self.query.LOUTER: - self.query.promote_alias_chain(alias_chain, True) - else: - alias = root_alias - - dedupe = f.column in opts.duplicate_targets - if dupe_set or dedupe: - avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ())) - if dedupe: - dupe_set.add((opts, f.column)) - - alias = self.query.join((alias, table, f.column, - f.rel.get_related_field().column), - exclusions=used.union(avoid), promote=promote) - used.add(alias) - columns, aliases = self.get_default_columns(start_alias=alias, - opts=f.rel.to._meta, as_pairs=True) - self.query.related_select_cols.extend(columns) - if self.query.alias_map[alias][JOIN_TYPE] == self.query.LOUTER: - self.query.promote_alias_chain(aliases, True) - self.query.related_select_fields.extend(f.rel.to._meta.fields) - if restricted: - next = requested.get(f.name, {}) - else: - next = False - new_nullable = f.null or promote - for dupe_opts, dupe_col in dupe_set: - self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) - self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, - used, next, restricted, new_nullable, dupe_set, avoid) - - if restricted: - related_fields = [ - (o.field, o.model) - for o in opts.get_all_related_objects() - if o.field.unique - ] - for f, model in related_fields: - if not select_related_descend(f, restricted, requested, reverse=True): - continue - # The "avoid" set is aliases we want to avoid just for this - # particular branch of the recursion. They aren't permanently - # forbidden from reuse in the related selection tables (which is - # what "used" specifies). - avoid = avoid_set.copy() - dupe_set = orig_dupe_set.copy() - table = model._meta.db_table - - int_opts = opts - alias = root_alias - alias_chain = [] - chain = opts.get_base_chain(f.rel.to) - if chain is not None: - for int_model in chain: - # Proxy model have elements in base chain - # with no parents, assign the new options - # object and skip to the next base in that - # case - if not int_opts.parents[int_model]: - int_opts = int_model._meta - continue - lhs_col = int_opts.parents[int_model].column - dedupe = lhs_col in opts.duplicate_targets - if dedupe: - avoid.update((self.query.dupe_avoidance.get(id(opts), lhs_col), - ())) - dupe_set.add((opts, lhs_col)) - int_opts = int_model._meta - alias = self.query.join( - (alias, int_opts.db_table, lhs_col, int_opts.pk.column), - exclusions=used, promote=True, reuse=used - ) - alias_chain.append(alias) - for dupe_opts, dupe_col in dupe_set: - self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) - dedupe = f.column in opts.duplicate_targets - if dupe_set or dedupe: - avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ())) - if dedupe: - dupe_set.add((opts, f.column)) - alias = self.query.join( - (alias, table, f.rel.get_related_field().column, f.column), - exclusions=used.union(avoid), - promote=True - ) - used.add(alias) - columns, aliases = self.get_default_columns(start_alias=alias, - opts=model._meta, as_pairs=True, local_only=True) - self.query.related_select_cols.extend(columns) - self.query.related_select_fields.extend(model._meta.fields) - - next = requested.get(f.related_query_name(), {}) - new_nullable = f.null or None - - self.fill_related_selections(model._meta, table, cur_depth+1, - used, next, restricted, new_nullable) - - def deferred_to_columns(self): - """ - Converts the self.deferred_loading data structure to mapping of table - names to sets of column names which are to be loaded. Returns the - dictionary. - """ - columns = {} - self.query.deferred_to_data(columns, self.query.deferred_to_columns_cb) - return columns - - def results_iter(self): - """ - Returns an iterator over the results from executing this query. - """ - resolve_columns = hasattr(self, 'resolve_columns') - fields = None - has_aggregate_select = bool(self.query.aggregate_select) - for rows in self.execute_sql(MULTI): - for row in rows: - if resolve_columns: - if fields is None: - # We only set this up here because - # related_select_fields isn't populated until - # execute_sql() has been called. - if self.query.select_fields: - fields = self.query.select_fields + self.query.related_select_fields - else: - fields = self.query.model._meta.fields - # If the field was deferred, exclude it from being passed - # into `resolve_columns` because it wasn't selected. - only_load = self.deferred_to_columns() - if only_load: - db_table = self.query.model._meta.db_table - fields = [f for f in fields if db_table in only_load and - f.column in only_load[db_table]] - row = self.resolve_columns(row, fields) - - if has_aggregate_select: - aggregate_start = len(self.query.extra_select.keys()) + len(self.query.select) - aggregate_end = aggregate_start + len(self.query.aggregate_select) - row = tuple(row[:aggregate_start]) + tuple([ - self.query.resolve_aggregate(value, aggregate, self.connection) - for (alias, aggregate), value - in zip(self.query.aggregate_select.items(), row[aggregate_start:aggregate_end]) - ]) + tuple(row[aggregate_end:]) - - yield row - - def execute_sql(self, result_type=MULTI): - """ - Run the query against the database and returns the result(s). The - return value is a single data item if result_type is SINGLE, or an - iterator over the results if the result_type is MULTI. - - result_type is either MULTI (use fetchmany() to retrieve all rows), - SINGLE (only retrieve a single row), or None. In this last case, the - cursor is returned if any query is executed, since it's used by - subclasses such as InsertQuery). It's possible, however, that no query - is needed, as the filters describe an empty set. In that case, None is - returned, to avoid any unnecessary database interaction. - """ - try: - sql, params = self.as_sql() - if not sql: - raise EmptyResultSet - except EmptyResultSet: - if result_type == MULTI: - return empty_iter() - else: - return - - cursor = self.connection.cursor() - cursor.execute(sql, params) - - if not result_type: - return cursor - if result_type == SINGLE: - if self.query.ordering_aliases: - return cursor.fetchone()[:-len(self.query.ordering_aliases)] - return cursor.fetchone() - - # The MULTI case. - if self.query.ordering_aliases: - result = order_modified_iter(cursor, len(self.query.ordering_aliases), - self.connection.features.empty_fetchmany_value) - else: - result = iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)), - self.connection.features.empty_fetchmany_value) - if not self.connection.features.can_use_chunked_reads: - # If we are using non-chunked reads, we return the same data - # structure as normally, but ensure it is all read into memory - # before going any further. - return list(result) - return result - - -class SQLInsertCompiler(SQLCompiler): - def placeholder(self, field, val): - if field is None: - # A field value of None means the value is raw. - return val - elif hasattr(field, 'get_placeholder'): - # Some fields (e.g. geo fields) need special munging before - # they can be inserted. - return field.get_placeholder(val, self.connection) - else: - # Return the common case for the placeholder - return '%s' - - def as_sql(self): - # We don't need quote_name_unless_alias() here, since these are all - # going to be column names (so we can avoid the extra overhead). - qn = self.connection.ops.quote_name - opts = self.query.model._meta - result = ['INSERT INTO %s' % qn(opts.db_table)] - result.append('(%s)' % ', '.join([qn(c) for c in self.query.columns])) - values = [self.placeholder(*v) for v in self.query.values] - result.append('VALUES (%s)' % ', '.join(values)) - params = self.query.params - if self.return_id and self.connection.features.can_return_id_from_insert: - col = "%s.%s" % (qn(opts.db_table), qn(opts.pk.column)) - r_fmt, r_params = self.connection.ops.return_insert_id() - result.append(r_fmt % col) - params = params + r_params - return ' '.join(result), params - - def execute_sql(self, return_id=False): - self.return_id = return_id - cursor = super(SQLInsertCompiler, self).execute_sql(None) - if not (return_id and cursor): - return - if self.connection.features.can_return_id_from_insert: - return self.connection.ops.fetch_returned_insert_id(cursor) - return self.connection.ops.last_insert_id(cursor, - self.query.model._meta.db_table, self.query.model._meta.pk.column) - - -class SQLDeleteCompiler(SQLCompiler): - def as_sql(self): - """ - Creates the SQL for this query. Returns the SQL string and list of - parameters. - """ - assert len(self.query.tables) == 1, \ - "Can only delete from one table at a time." - qn = self.quote_name_unless_alias - result = ['DELETE FROM %s' % qn(self.query.tables[0])] - where, params = self.query.where.as_sql(qn=qn, connection=self.connection) - result.append('WHERE %s' % where) - return ' '.join(result), tuple(params) - -class SQLUpdateCompiler(SQLCompiler): - def as_sql(self): - """ - Creates the SQL for this query. Returns the SQL string and list of - parameters. - """ - from django.db.models.base import Model - - self.pre_sql_setup() - if not self.query.values: - return '', () - table = self.query.tables[0] - qn = self.quote_name_unless_alias - result = ['UPDATE %s' % qn(table)] - result.append('SET') - values, update_params = [], [] - for field, model, val in self.query.values: - if hasattr(val, 'prepare_database_save'): - val = val.prepare_database_save(field) - else: - val = field.get_db_prep_save(val, connection=self.connection) - - # Getting the placeholder for the field. - if hasattr(field, 'get_placeholder'): - placeholder = field.get_placeholder(val, self.connection) - else: - placeholder = '%s' - - if hasattr(val, 'evaluate'): - val = SQLEvaluator(val, self.query, allow_joins=False) - name = field.column - if hasattr(val, 'as_sql'): - sql, params = val.as_sql(qn, self.connection) - values.append('%s = %s' % (qn(name), sql)) - update_params.extend(params) - elif val is not None: - values.append('%s = %s' % (qn(name), placeholder)) - update_params.append(val) - else: - values.append('%s = NULL' % qn(name)) - if not values: - return '', () - result.append(', '.join(values)) - where, params = self.query.where.as_sql(qn=qn, connection=self.connection) - if where: - result.append('WHERE %s' % where) - return ' '.join(result), tuple(update_params + params) - - def execute_sql(self, result_type): - """ - Execute the specified update. Returns the number of rows affected by - the primary update query. The "primary update query" is the first - non-empty query that is executed. Row counts for any subsequent, - related queries are not available. - """ - cursor = super(SQLUpdateCompiler, self).execute_sql(result_type) - rows = cursor and cursor.rowcount or 0 - is_empty = cursor is None - del cursor - for query in self.query.get_related_updates(): - aux_rows = query.get_compiler(self.using).execute_sql(result_type) - if is_empty: - rows = aux_rows - is_empty = False - return rows - - def pre_sql_setup(self): - """ - If the update depends on results from other tables, we need to do some - munging of the "where" conditions to match the format required for - (portable) SQL updates. That is done here. - - Further, if we are going to be running multiple updates, we pull out - the id values to update at this point so that they don't change as a - result of the progressive updates. - """ - self.query.select_related = False - self.query.clear_ordering(True) - super(SQLUpdateCompiler, self).pre_sql_setup() - count = self.query.count_active_tables() - if not self.query.related_updates and count == 1: - return - - # We need to use a sub-select in the where clause to filter on things - # from other tables. - query = self.query.clone(klass=Query) - query.bump_prefix() - query.extra = {} - query.select = [] - query.add_fields([query.model._meta.pk.name]) - must_pre_select = count > 1 and not self.connection.features.update_can_self_select - - # Now we adjust the current query: reset the where clause and get rid - # of all the tables we don't need (since they're in the sub-select). - self.query.where = self.query.where_class() - if self.query.related_updates or must_pre_select: - # Either we're using the idents in multiple update queries (so - # don't want them to change), or the db backend doesn't support - # selecting from the updating table (e.g. MySQL). - idents = [] - for rows in query.get_compiler(self.using).execute_sql(MULTI): - idents.extend([r[0] for r in rows]) - self.query.add_filter(('pk__in', idents)) - self.query.related_ids = idents - else: - # The fast path. Filters and updates in one query. - self.query.add_filter(('pk__in', query)) - for alias in self.query.tables[1:]: - self.query.alias_refcount[alias] = 0 - -class SQLAggregateCompiler(SQLCompiler): - def as_sql(self, qn=None): - """ - Creates the SQL for this query. Returns the SQL string and list of - parameters. - """ - if qn is None: - qn = self.quote_name_unless_alias - sql = ('SELECT %s FROM (%s) subquery' % ( - ', '.join([ - aggregate.as_sql(qn, self.connection) - for aggregate in self.query.aggregate_select.values() - ]), - self.query.subquery) - ) - params = self.query.sub_params - return (sql, params) - -class SQLDateCompiler(SQLCompiler): - def results_iter(self): - """ - Returns an iterator over the results from executing this query. - """ - resolve_columns = hasattr(self, 'resolve_columns') - if resolve_columns: - from django.db.models.fields import DateTimeField - fields = [DateTimeField()] - else: - from django.db.backends.util import typecast_timestamp - needs_string_cast = self.connection.features.needs_datetime_string_cast - - offset = len(self.query.extra_select) - for rows in self.execute_sql(MULTI): - for row in rows: - date = row[offset] - if resolve_columns: - date = self.resolve_columns(row, fields)[offset] - elif needs_string_cast: - date = typecast_timestamp(str(date)) - yield date - - -def empty_iter(): - """ - Returns an iterator containing no results. - """ - yield iter([]).next() - - -def order_modified_iter(cursor, trim, sentinel): - """ - Yields blocks of rows from a cursor. We use this iterator in the special - case when extra output columns have been added to support ordering - requirements. We must trim those extra columns before anything else can use - the results, since they're only needed to make the SQL valid. - """ - for rows in iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)), - sentinel): - yield [r[:-trim] for r in rows] diff --git a/parts/django/django/db/models/sql/constants.py b/parts/django/django/db/models/sql/constants.py deleted file mode 100644 index 63c704f..0000000 --- a/parts/django/django/db/models/sql/constants.py +++ /dev/null @@ -1,37 +0,0 @@ -import re - -# Valid query types (a dictionary is used for speedy lookups). -QUERY_TERMS = dict([(x, None) for x in ( - 'exact', 'iexact', 'contains', 'icontains', 'gt', 'gte', 'lt', 'lte', 'in', - 'startswith', 'istartswith', 'endswith', 'iendswith', 'range', 'year', - 'month', 'day', 'week_day', 'isnull', 'search', 'regex', 'iregex', - )]) - -# Size of each "chunk" for get_iterator calls. -# Larger values are slightly faster at the expense of more storage space. -GET_ITERATOR_CHUNK_SIZE = 100 - -# Separator used to split filter strings apart. -LOOKUP_SEP = '__' - -# Constants to make looking up tuple values clearer. -# Join lists (indexes into the tuples that are values in the alias_map -# dictionary in the Query class). -TABLE_NAME = 0 -RHS_ALIAS = 1 -JOIN_TYPE = 2 -LHS_ALIAS = 3 -LHS_JOIN_COL = 4 -RHS_JOIN_COL = 5 -NULLABLE = 6 - -# How many results to expect from a cursor.execute call -MULTI = 'multi' -SINGLE = 'single' - -ORDER_PATTERN = re.compile(r'\?|[-+]?[.\w]+$') -ORDER_DIR = { - 'ASC': ('ASC', 'DESC'), - 'DESC': ('DESC', 'ASC')} - - diff --git a/parts/django/django/db/models/sql/datastructures.py b/parts/django/django/db/models/sql/datastructures.py deleted file mode 100644 index 92d64e1..0000000 --- a/parts/django/django/db/models/sql/datastructures.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Useful auxilliary data structures for query construction. Not useful outside -the SQL domain. -""" - -class EmptyResultSet(Exception): - pass - -class FullResultSet(Exception): - pass - -class MultiJoin(Exception): - """ - Used by join construction code to indicate the point at which a - multi-valued join was attempted (if the caller wants to treat that - exceptionally). - """ - def __init__(self, level): - self.level = level - -class Empty(object): - pass - -class RawValue(object): - def __init__(self, value): - self.value = value - -class Date(object): - """ - Add a date selection column. - """ - def __init__(self, col, lookup_type): - self.col = col - self.lookup_type = lookup_type - - def relabel_aliases(self, change_map): - c = self.col - if isinstance(c, (list, tuple)): - self.col = (change_map.get(c[0], c[0]), c[1]) - - def as_sql(self, qn, connection): - if isinstance(self.col, (list, tuple)): - col = '%s.%s' % tuple([qn(c) for c in self.col]) - else: - col = self.col - return connection.ops.date_trunc_sql(self.lookup_type, col) diff --git a/parts/django/django/db/models/sql/expressions.py b/parts/django/django/db/models/sql/expressions.py deleted file mode 100644 index fffbba0..0000000 --- a/parts/django/django/db/models/sql/expressions.py +++ /dev/null @@ -1,87 +0,0 @@ -from django.core.exceptions import FieldError -from django.db.models.fields import FieldDoesNotExist -from django.db.models.sql.constants import LOOKUP_SEP - -class SQLEvaluator(object): - def __init__(self, expression, query, allow_joins=True): - self.expression = expression - self.opts = query.get_meta() - self.cols = {} - - self.contains_aggregate = False - self.expression.prepare(self, query, allow_joins) - - def prepare(self): - return self - - def as_sql(self, qn, connection): - return self.expression.evaluate(self, qn, connection) - - def relabel_aliases(self, change_map): - for node, col in self.cols.items(): - if hasattr(col, "relabel_aliases"): - col.relabel_aliases(change_map) - else: - self.cols[node] = (change_map.get(col[0], col[0]), col[1]) - - ##################################################### - # Vistor methods for initial expression preparation # - ##################################################### - - def prepare_node(self, node, query, allow_joins): - for child in node.children: - if hasattr(child, 'prepare'): - child.prepare(self, query, allow_joins) - - def prepare_leaf(self, node, query, allow_joins): - if not allow_joins and LOOKUP_SEP in node.name: - raise FieldError("Joined field references are not permitted in this query") - - field_list = node.name.split(LOOKUP_SEP) - if (len(field_list) == 1 and - node.name in query.aggregate_select.keys()): - self.contains_aggregate = True - self.cols[node] = query.aggregate_select[node.name] - else: - try: - field, source, opts, join_list, last, _ = query.setup_joins( - field_list, query.get_meta(), - query.get_initial_alias(), False) - col, _, join_list = query.trim_joins(source, join_list, last, False) - - self.cols[node] = (join_list[-1], col) - except FieldDoesNotExist: - raise FieldError("Cannot resolve keyword %r into field. " - "Choices are: %s" % (self.name, - [f.name for f in self.opts.fields])) - - ################################################## - # Vistor methods for final expression evaluation # - ################################################## - - def evaluate_node(self, node, qn, connection): - expressions = [] - expression_params = [] - for child in node.children: - if hasattr(child, 'evaluate'): - sql, params = child.evaluate(self, qn, connection) - else: - sql, params = '%s', (child,) - - if len(getattr(child, 'children', [])) > 1: - format = '(%s)' - else: - format = '%s' - - if sql: - expressions.append(format % sql) - expression_params.extend(params) - - return connection.ops.combine_expression(node.connector, expressions), expression_params - - def evaluate_leaf(self, node, qn, connection): - col = self.cols[node] - if hasattr(col, 'as_sql'): - return col.as_sql(qn, connection), () - else: - return '%s.%s' % (qn(col[0]), qn(col[1])), () diff --git a/parts/django/django/db/models/sql/query.py b/parts/django/django/db/models/sql/query.py deleted file mode 100644 index eae7a87..0000000 --- a/parts/django/django/db/models/sql/query.py +++ /dev/null @@ -1,1860 +0,0 @@ -""" -Create SQL statements for QuerySets. - -The code in here encapsulates all of the SQL construction so that QuerySets -themselves do not have to (and could be backed by things other than SQL -databases). The abstraction barrier only works one way: this module has to know -all about the internals of models in order to get the information it needs. -""" - -from django.utils.copycompat import deepcopy -from django.utils.tree import Node -from django.utils.datastructures import SortedDict -from django.utils.encoding import force_unicode -from django.db import connections, DEFAULT_DB_ALIAS -from django.db.models import signals -from django.db.models.fields import FieldDoesNotExist -from django.db.models.query_utils import select_related_descend, InvalidQuery -from django.db.models.sql import aggregates as base_aggregates_module -from django.db.models.sql.constants import * -from django.db.models.sql.datastructures import EmptyResultSet, Empty, MultiJoin -from django.db.models.sql.expressions import SQLEvaluator -from django.db.models.sql.where import (WhereNode, Constraint, EverythingNode, - ExtraWhere, AND, OR) -from django.core.exceptions import FieldError - -__all__ = ['Query', 'RawQuery'] - -class RawQuery(object): - """ - A single raw SQL query - """ - - def __init__(self, sql, using, params=None): - self.validate_sql(sql) - self.params = params or () - self.sql = sql - self.using = using - self.cursor = None - - # Mirror some properties of a normal query so that - # the compiler can be used to process results. - self.low_mark, self.high_mark = 0, None # Used for offset/limit - self.extra_select = {} - self.aggregate_select = {} - - def clone(self, using): - return RawQuery(self.sql, using, params=self.params) - - def convert_values(self, value, field, connection): - """Convert the database-returned value into a type that is consistent - across database backends. - - By default, this defers to the underlying backend operations, but - it can be overridden by Query classes for specific backends. - """ - return connection.ops.convert_values(value, field) - - def get_columns(self): - if self.cursor is None: - self._execute_query() - converter = connections[self.using].introspection.table_name_converter - return [converter(column_meta[0]) - for column_meta in self.cursor.description] - - def validate_sql(self, sql): - if not sql.lower().strip().startswith('select'): - raise InvalidQuery('Raw queries are limited to SELECT queries. Use ' - 'connection.cursor directly for other types of queries.') - - def __iter__(self): - # Always execute a new query for a new iterator. - # This could be optimized with a cache at the expense of RAM. - self._execute_query() - if not connections[self.using].features.can_use_chunked_reads: - # If the database can't use chunked reads we need to make sure we - # evaluate the entire query up front. - result = list(self.cursor) - else: - result = self.cursor - return iter(result) - - def __repr__(self): - return "<RawQuery: %r>" % (self.sql % self.params) - - def _execute_query(self): - self.cursor = connections[self.using].cursor() - self.cursor.execute(self.sql, self.params) - - -class Query(object): - """ - A single SQL query. - """ - # SQL join types. These are part of the class because their string forms - # vary from database to database and can be customised by a subclass. - INNER = 'INNER JOIN' - LOUTER = 'LEFT OUTER JOIN' - - alias_prefix = 'T' - query_terms = QUERY_TERMS - aggregates_module = base_aggregates_module - - compiler = 'SQLCompiler' - - def __init__(self, model, where=WhereNode): - self.model = model - self.alias_refcount = {} - self.alias_map = {} # Maps alias to join information - self.table_map = {} # Maps table names to list of aliases. - self.join_map = {} - self.rev_join_map = {} # Reverse of join_map. - self.quote_cache = {} - self.default_cols = True - self.default_ordering = True - self.standard_ordering = True - self.ordering_aliases = [] - self.select_fields = [] - self.related_select_fields = [] - self.dupe_avoidance = {} - self.used_aliases = set() - self.filter_is_sticky = False - self.included_inherited_models = {} - - # SQL-related attributes - self.select = [] - self.tables = [] # Aliases in the order they are created. - self.where = where() - self.where_class = where - self.group_by = None - self.having = where() - self.order_by = [] - self.low_mark, self.high_mark = 0, None # Used for offset/limit - self.distinct = False - self.select_related = False - self.related_select_cols = [] - - # SQL aggregate-related attributes - self.aggregates = SortedDict() # Maps alias -> SQL aggregate function - self.aggregate_select_mask = None - self._aggregate_select_cache = None - - # Arbitrary maximum limit for select_related. Prevents infinite - # recursion. Can be changed by the depth parameter to select_related(). - self.max_depth = 5 - - # These are for extensions. The contents are more or less appended - # verbatim to the appropriate clause. - self.extra = SortedDict() # Maps col_alias -> (col_sql, params). - self.extra_select_mask = None - self._extra_select_cache = None - - self.extra_tables = () - self.extra_order_by = () - - # A tuple that is a set of model field names and either True, if these - # are the fields to defer, or False if these are the only fields to - # load. - self.deferred_loading = (set(), True) - - def __str__(self): - """ - Returns the query as a string of SQL with the parameter values - substituted in. - - Parameter values won't necessarily be quoted correctly, since that is - done by the database interface at execution time. - """ - sql, params = self.get_compiler(DEFAULT_DB_ALIAS).as_sql() - return sql % params - - def __deepcopy__(self, memo): - result = self.clone(memo=memo) - memo[id(self)] = result - return result - - def __getstate__(self): - """ - Pickling support. - """ - obj_dict = self.__dict__.copy() - obj_dict['related_select_fields'] = [] - obj_dict['related_select_cols'] = [] - - # Fields can't be pickled, so if a field list has been - # specified, we pickle the list of field names instead. - # None is also a possible value; that can pass as-is - obj_dict['select_fields'] = [ - f is not None and f.name or None - for f in obj_dict['select_fields'] - ] - return obj_dict - - def __setstate__(self, obj_dict): - """ - Unpickling support. - """ - # Rebuild list of field instances - opts = obj_dict['model']._meta - obj_dict['select_fields'] = [ - name is not None and opts.get_field(name) or None - for name in obj_dict['select_fields'] - ] - - self.__dict__.update(obj_dict) - - def prepare(self): - return self - - def get_compiler(self, using=None, connection=None): - if using is None and connection is None: - raise ValueError("Need either using or connection") - if using: - connection = connections[using] - - # Check that the compiler will be able to execute the query - for alias, aggregate in self.aggregate_select.items(): - connection.ops.check_aggregate_support(aggregate) - - return connection.ops.compiler(self.compiler)(self, connection, using) - - def get_meta(self): - """ - Returns the Options instance (the model._meta) from which to start - processing. Normally, this is self.model._meta, but it can be changed - by subclasses. - """ - return self.model._meta - - def clone(self, klass=None, memo=None, **kwargs): - """ - Creates a copy of the current instance. The 'kwargs' parameter can be - used by clients to update attributes after copying has taken place. - """ - obj = Empty() - obj.__class__ = klass or self.__class__ - obj.model = self.model - obj.alias_refcount = self.alias_refcount.copy() - obj.alias_map = self.alias_map.copy() - obj.table_map = self.table_map.copy() - obj.join_map = self.join_map.copy() - obj.rev_join_map = self.rev_join_map.copy() - obj.quote_cache = {} - obj.default_cols = self.default_cols - obj.default_ordering = self.default_ordering - obj.standard_ordering = self.standard_ordering - obj.included_inherited_models = self.included_inherited_models.copy() - obj.ordering_aliases = [] - obj.select_fields = self.select_fields[:] - obj.related_select_fields = self.related_select_fields[:] - obj.dupe_avoidance = self.dupe_avoidance.copy() - obj.select = self.select[:] - obj.tables = self.tables[:] - obj.where = deepcopy(self.where, memo=memo) - obj.where_class = self.where_class - if self.group_by is None: - obj.group_by = None - else: - obj.group_by = self.group_by[:] - obj.having = deepcopy(self.having, memo=memo) - obj.order_by = self.order_by[:] - obj.low_mark, obj.high_mark = self.low_mark, self.high_mark - obj.distinct = self.distinct - obj.select_related = self.select_related - obj.related_select_cols = [] - obj.aggregates = deepcopy(self.aggregates, memo=memo) - if self.aggregate_select_mask is None: - obj.aggregate_select_mask = None - else: - obj.aggregate_select_mask = self.aggregate_select_mask.copy() - # _aggregate_select_cache cannot be copied, as doing so breaks the - # (necessary) state in which both aggregates and - # _aggregate_select_cache point to the same underlying objects. - # It will get re-populated in the cloned queryset the next time it's - # used. - obj._aggregate_select_cache = None - obj.max_depth = self.max_depth - obj.extra = self.extra.copy() - if self.extra_select_mask is None: - obj.extra_select_mask = None - else: - obj.extra_select_mask = self.extra_select_mask.copy() - if self._extra_select_cache is None: - obj._extra_select_cache = None - else: - obj._extra_select_cache = self._extra_select_cache.copy() - obj.extra_tables = self.extra_tables - obj.extra_order_by = self.extra_order_by - obj.deferred_loading = deepcopy(self.deferred_loading, memo=memo) - if self.filter_is_sticky and self.used_aliases: - obj.used_aliases = self.used_aliases.copy() - else: - obj.used_aliases = set() - obj.filter_is_sticky = False - obj.__dict__.update(kwargs) - if hasattr(obj, '_setup_query'): - obj._setup_query() - return obj - - def convert_values(self, value, field, connection): - """Convert the database-returned value into a type that is consistent - across database backends. - - By default, this defers to the underlying backend operations, but - it can be overridden by Query classes for specific backends. - """ - return connection.ops.convert_values(value, field) - - def resolve_aggregate(self, value, aggregate, connection): - """Resolve the value of aggregates returned by the database to - consistent (and reasonable) types. - - This is required because of the predisposition of certain backends - to return Decimal and long types when they are not needed. - """ - if value is None: - if aggregate.is_ordinal: - return 0 - # Return None as-is - return value - elif aggregate.is_ordinal: - # Any ordinal aggregate (e.g., count) returns an int - return int(value) - elif aggregate.is_computed: - # Any computed aggregate (e.g., avg) returns a float - return float(value) - else: - # Return value depends on the type of the field being processed. - return self.convert_values(value, aggregate.field, connection) - - def get_aggregation(self, using): - """ - Returns the dictionary with the values of the existing aggregations. - """ - if not self.aggregate_select: - return {} - - # If there is a group by clause, aggregating does not add useful - # information but retrieves only the first row. Aggregate - # over the subquery instead. - if self.group_by is not None: - from django.db.models.sql.subqueries import AggregateQuery - query = AggregateQuery(self.model) - - obj = self.clone() - - # Remove any aggregates marked for reduction from the subquery - # and move them to the outer AggregateQuery. - for alias, aggregate in self.aggregate_select.items(): - if aggregate.is_summary: - query.aggregate_select[alias] = aggregate - del obj.aggregate_select[alias] - - try: - query.add_subquery(obj, using) - except EmptyResultSet: - return dict( - (alias, None) - for alias in query.aggregate_select - ) - else: - query = self - self.select = [] - self.default_cols = False - self.extra = {} - self.remove_inherited_models() - - query.clear_ordering(True) - query.clear_limits() - query.select_related = False - query.related_select_cols = [] - query.related_select_fields = [] - - result = query.get_compiler(using).execute_sql(SINGLE) - if result is None: - result = [None for q in query.aggregate_select.items()] - - return dict([ - (alias, self.resolve_aggregate(val, aggregate, connection=connections[using])) - for (alias, aggregate), val - in zip(query.aggregate_select.items(), result) - ]) - - def get_count(self, using): - """ - Performs a COUNT() query using the current filter constraints. - """ - obj = self.clone() - if len(self.select) > 1 or self.aggregate_select: - # If a select clause exists, then the query has already started to - # specify the columns that are to be returned. - # In this case, we need to use a subquery to evaluate the count. - from django.db.models.sql.subqueries import AggregateQuery - subquery = obj - subquery.clear_ordering(True) - subquery.clear_limits() - - obj = AggregateQuery(obj.model) - try: - obj.add_subquery(subquery, using=using) - except EmptyResultSet: - # add_subquery evaluates the query, if it's an EmptyResultSet - # then there are can be no results, and therefore there the - # count is obviously 0 - return 0 - - obj.add_count_column() - number = obj.get_aggregation(using=using)[None] - - # Apply offset and limit constraints manually, since using LIMIT/OFFSET - # in SQL (in variants that provide them) doesn't change the COUNT - # output. - number = max(0, number - self.low_mark) - if self.high_mark is not None: - number = min(number, self.high_mark - self.low_mark) - - return number - - def has_results(self, using): - q = self.clone() - q.add_extra({'a': 1}, None, None, None, None, None) - q.select = [] - q.select_fields = [] - q.default_cols = False - q.select_related = False - q.set_extra_mask(('a',)) - q.set_aggregate_mask(()) - q.clear_ordering(True) - q.set_limits(high=1) - compiler = q.get_compiler(using=using) - return bool(compiler.execute_sql(SINGLE)) - - def combine(self, rhs, connector): - """ - Merge the 'rhs' query into the current one (with any 'rhs' effects - being applied *after* (that is, "to the right of") anything in the - current query. 'rhs' is not modified during a call to this function. - - The 'connector' parameter describes how to connect filters from the - 'rhs' query. - """ - assert self.model == rhs.model, \ - "Cannot combine queries on two different base models." - assert self.can_filter(), \ - "Cannot combine queries once a slice has been taken." - assert self.distinct == rhs.distinct, \ - "Cannot combine a unique query with a non-unique query." - - self.remove_inherited_models() - # Work out how to relabel the rhs aliases, if necessary. - change_map = {} - used = set() - conjunction = (connector == AND) - first = True - for alias in rhs.tables: - if not rhs.alias_refcount[alias]: - # An unused alias. - continue - promote = (rhs.alias_map[alias][JOIN_TYPE] == self.LOUTER) - new_alias = self.join(rhs.rev_join_map[alias], - (conjunction and not first), used, promote, not conjunction) - used.add(new_alias) - change_map[alias] = new_alias - first = False - - # So that we don't exclude valid results in an "or" query combination, - # the first join that is exclusive to the lhs (self) must be converted - # to an outer join. - if not conjunction: - for alias in self.tables[1:]: - if self.alias_refcount[alias] == 1: - self.promote_alias(alias, True) - break - - # Now relabel a copy of the rhs where-clause and add it to the current - # one. - if rhs.where: - w = deepcopy(rhs.where) - w.relabel_aliases(change_map) - if not self.where: - # Since 'self' matches everything, add an explicit "include - # everything" where-constraint so that connections between the - # where clauses won't exclude valid results. - self.where.add(EverythingNode(), AND) - elif self.where: - # rhs has an empty where clause. - w = self.where_class() - w.add(EverythingNode(), AND) - else: - w = self.where_class() - self.where.add(w, connector) - - # Selection columns and extra extensions are those provided by 'rhs'. - self.select = [] - for col in rhs.select: - if isinstance(col, (list, tuple)): - self.select.append((change_map.get(col[0], col[0]), col[1])) - else: - item = deepcopy(col) - item.relabel_aliases(change_map) - self.select.append(item) - self.select_fields = rhs.select_fields[:] - - if connector == OR: - # It would be nice to be able to handle this, but the queries don't - # really make sense (or return consistent value sets). Not worth - # the extra complexity when you can write a real query instead. - if self.extra and rhs.extra: - raise ValueError("When merging querysets using 'or', you " - "cannot have extra(select=...) on both sides.") - self.extra.update(rhs.extra) - extra_select_mask = set() - if self.extra_select_mask is not None: - extra_select_mask.update(self.extra_select_mask) - if rhs.extra_select_mask is not None: - extra_select_mask.update(rhs.extra_select_mask) - if extra_select_mask: - self.set_extra_mask(extra_select_mask) - self.extra_tables += rhs.extra_tables - - # Ordering uses the 'rhs' ordering, unless it has none, in which case - # the current ordering is used. - self.order_by = rhs.order_by and rhs.order_by[:] or self.order_by - self.extra_order_by = rhs.extra_order_by or self.extra_order_by - - def deferred_to_data(self, target, callback): - """ - Converts the self.deferred_loading data structure to an alternate data - structure, describing the field that *will* be loaded. This is used to - compute the columns to select from the database and also by the - QuerySet class to work out which fields are being initialised on each - model. Models that have all their fields included aren't mentioned in - the result, only those that have field restrictions in place. - - The "target" parameter is the instance that is populated (in place). - The "callback" is a function that is called whenever a (model, field) - pair need to be added to "target". It accepts three parameters: - "target", and the model and list of fields being added for that model. - """ - field_names, defer = self.deferred_loading - if not field_names: - return - columns = set() - orig_opts = self.model._meta - seen = {} - must_include = {self.model: set([orig_opts.pk])} - for field_name in field_names: - parts = field_name.split(LOOKUP_SEP) - cur_model = self.model - opts = orig_opts - for name in parts[:-1]: - old_model = cur_model - source = opts.get_field_by_name(name)[0] - cur_model = opts.get_field_by_name(name)[0].rel.to - opts = cur_model._meta - # Even if we're "just passing through" this model, we must add - # both the current model's pk and the related reference field - # to the things we select. - must_include[old_model].add(source) - add_to_dict(must_include, cur_model, opts.pk) - field, model, _, _ = opts.get_field_by_name(parts[-1]) - if model is None: - model = cur_model - add_to_dict(seen, model, field) - - if defer: - # We need to load all fields for each model, except those that - # appear in "seen" (for all models that appear in "seen"). The only - # slight complexity here is handling fields that exist on parent - # models. - workset = {} - for model, values in seen.iteritems(): - for field, m in model._meta.get_fields_with_model(): - if field in values: - continue - add_to_dict(workset, m or model, field) - for model, values in must_include.iteritems(): - # If we haven't included a model in workset, we don't add the - # corresponding must_include fields for that model, since an - # empty set means "include all fields". That's why there's no - # "else" branch here. - if model in workset: - workset[model].update(values) - for model, values in workset.iteritems(): - callback(target, model, values) - else: - for model, values in must_include.iteritems(): - if model in seen: - seen[model].update(values) - else: - # As we've passed through this model, but not explicitly - # included any fields, we have to make sure it's mentioned - # so that only the "must include" fields are pulled in. - seen[model] = values - # Now ensure that every model in the inheritance chain is mentioned - # in the parent list. Again, it must be mentioned to ensure that - # only "must include" fields are pulled in. - for model in orig_opts.get_parent_list(): - if model not in seen: - seen[model] = set() - for model, values in seen.iteritems(): - callback(target, model, values) - - - def deferred_to_columns_cb(self, target, model, fields): - """ - Callback used by deferred_to_columns(). The "target" parameter should - be a set instance. - """ - table = model._meta.db_table - if table not in target: - target[table] = set() - for field in fields: - target[table].add(field.column) - - - def table_alias(self, table_name, create=False): - """ - Returns a table alias for the given table_name and whether this is a - new alias or not. - - If 'create' is true, a new alias is always created. Otherwise, the - most recently created alias for the table (if one exists) is reused. - """ - current = self.table_map.get(table_name) - if not create and current: - alias = current[0] - self.alias_refcount[alias] += 1 - return alias, False - - # Create a new alias for this table. - if current: - alias = '%s%d' % (self.alias_prefix, len(self.alias_map) + 1) - current.append(alias) - else: - # The first occurence of a table uses the table name directly. - alias = table_name - self.table_map[alias] = [alias] - self.alias_refcount[alias] = 1 - self.tables.append(alias) - return alias, True - - def ref_alias(self, alias): - """ Increases the reference count for this alias. """ - self.alias_refcount[alias] += 1 - - def unref_alias(self, alias): - """ Decreases the reference count for this alias. """ - self.alias_refcount[alias] -= 1 - - def promote_alias(self, alias, unconditional=False): - """ - Promotes the join type of an alias to an outer join if it's possible - for the join to contain NULL values on the left. If 'unconditional' is - False, the join is only promoted if it is nullable, otherwise it is - always promoted. - - Returns True if the join was promoted. - """ - if ((unconditional or self.alias_map[alias][NULLABLE]) and - self.alias_map[alias][JOIN_TYPE] != self.LOUTER): - data = list(self.alias_map[alias]) - data[JOIN_TYPE] = self.LOUTER - self.alias_map[alias] = tuple(data) - return True - return False - - def promote_alias_chain(self, chain, must_promote=False): - """ - Walks along a chain of aliases, promoting the first nullable join and - any joins following that. If 'must_promote' is True, all the aliases in - the chain are promoted. - """ - for alias in chain: - if self.promote_alias(alias, must_promote): - must_promote = True - - def promote_unused_aliases(self, initial_refcounts, used_aliases): - """ - Given a "before" copy of the alias_refcounts dictionary (as - 'initial_refcounts') and a collection of aliases that may have been - changed or created, works out which aliases have been created since - then and which ones haven't been used and promotes all of those - aliases, plus any children of theirs in the alias tree, to outer joins. - """ - # FIXME: There's some (a lot of!) overlap with the similar OR promotion - # in add_filter(). It's not quite identical, but is very similar. So - # pulling out the common bits is something for later. - considered = {} - for alias in self.tables: - if alias not in used_aliases: - continue - if (alias not in initial_refcounts or - self.alias_refcount[alias] == initial_refcounts[alias]): - parent = self.alias_map[alias][LHS_ALIAS] - must_promote = considered.get(parent, False) - promoted = self.promote_alias(alias, must_promote) - considered[alias] = must_promote or promoted - - def change_aliases(self, change_map): - """ - Changes the aliases in change_map (which maps old-alias -> new-alias), - relabelling any references to them in select columns and the where - clause. - """ - assert set(change_map.keys()).intersection(set(change_map.values())) == set() - - # 1. Update references in "select" (normal columns plus aliases), - # "group by", "where" and "having". - self.where.relabel_aliases(change_map) - self.having.relabel_aliases(change_map) - for columns in [self.select, self.group_by or []]: - for pos, col in enumerate(columns): - if isinstance(col, (list, tuple)): - old_alias = col[0] - columns[pos] = (change_map.get(old_alias, old_alias), col[1]) - else: - col.relabel_aliases(change_map) - for mapping in [self.aggregates]: - for key, col in mapping.items(): - if isinstance(col, (list, tuple)): - old_alias = col[0] - mapping[key] = (change_map.get(old_alias, old_alias), col[1]) - else: - col.relabel_aliases(change_map) - - # 2. Rename the alias in the internal table/alias datastructures. - for old_alias, new_alias in change_map.iteritems(): - alias_data = list(self.alias_map[old_alias]) - alias_data[RHS_ALIAS] = new_alias - - t = self.rev_join_map[old_alias] - data = list(self.join_map[t]) - data[data.index(old_alias)] = new_alias - self.join_map[t] = tuple(data) - self.rev_join_map[new_alias] = t - del self.rev_join_map[old_alias] - self.alias_refcount[new_alias] = self.alias_refcount[old_alias] - del self.alias_refcount[old_alias] - self.alias_map[new_alias] = tuple(alias_data) - del self.alias_map[old_alias] - - table_aliases = self.table_map[alias_data[TABLE_NAME]] - for pos, alias in enumerate(table_aliases): - if alias == old_alias: - table_aliases[pos] = new_alias - break - for pos, alias in enumerate(self.tables): - if alias == old_alias: - self.tables[pos] = new_alias - break - for key, alias in self.included_inherited_models.items(): - if alias in change_map: - self.included_inherited_models[key] = change_map[alias] - - # 3. Update any joins that refer to the old alias. - for alias, data in self.alias_map.iteritems(): - lhs = data[LHS_ALIAS] - if lhs in change_map: - data = list(data) - data[LHS_ALIAS] = change_map[lhs] - self.alias_map[alias] = tuple(data) - - def bump_prefix(self, exceptions=()): - """ - Changes the alias prefix to the next letter in the alphabet and - relabels all the aliases. Even tables that previously had no alias will - get an alias after this call (it's mostly used for nested queries and - the outer query will already be using the non-aliased table name). - - Subclasses who create their own prefix should override this method to - produce a similar result (a new prefix and relabelled aliases). - - The 'exceptions' parameter is a container that holds alias names which - should not be changed. - """ - current = ord(self.alias_prefix) - assert current < ord('Z') - prefix = chr(current + 1) - self.alias_prefix = prefix - change_map = {} - for pos, alias in enumerate(self.tables): - if alias in exceptions: - continue - new_alias = '%s%d' % (prefix, pos) - change_map[alias] = new_alias - self.tables[pos] = new_alias - self.change_aliases(change_map) - - def get_initial_alias(self): - """ - Returns the first alias for this query, after increasing its reference - count. - """ - if self.tables: - alias = self.tables[0] - self.ref_alias(alias) - else: - alias = self.join((None, self.model._meta.db_table, None, None)) - return alias - - def count_active_tables(self): - """ - Returns the number of tables in this query with a non-zero reference - count. - """ - return len([1 for count in self.alias_refcount.itervalues() if count]) - - def join(self, connection, always_create=False, exclusions=(), - promote=False, outer_if_first=False, nullable=False, reuse=None): - """ - Returns an alias for the join in 'connection', either reusing an - existing alias for that join or creating a new one. 'connection' is a - tuple (lhs, table, lhs_col, col) where 'lhs' is either an existing - table alias or a table name. The join correspods to the SQL equivalent - of:: - - lhs.lhs_col = table.col - - If 'always_create' is True and 'reuse' is None, a new alias is always - created, regardless of whether one already exists or not. If - 'always_create' is True and 'reuse' is a set, an alias in 'reuse' that - matches the connection will be returned, if possible. If - 'always_create' is False, the first existing alias that matches the - 'connection' is returned, if any. Otherwise a new join is created. - - If 'exclusions' is specified, it is something satisfying the container - protocol ("foo in exclusions" must work) and specifies a list of - aliases that should not be returned, even if they satisfy the join. - - If 'promote' is True, the join type for the alias will be LOUTER (if - the alias previously existed, the join type will be promoted from INNER - to LOUTER, if necessary). - - If 'outer_if_first' is True and a new join is created, it will have the - LOUTER join type. This is used when joining certain types of querysets - and Q-objects together. - - If 'nullable' is True, the join can potentially involve NULL values and - is a candidate for promotion (to "left outer") when combining querysets. - """ - lhs, table, lhs_col, col = connection - if lhs in self.alias_map: - lhs_table = self.alias_map[lhs][TABLE_NAME] - else: - lhs_table = lhs - - if reuse and always_create and table in self.table_map: - # Convert the 'reuse' to case to be "exclude everything but the - # reusable set, minus exclusions, for this table". - exclusions = set(self.table_map[table]).difference(reuse).union(set(exclusions)) - always_create = False - t_ident = (lhs_table, table, lhs_col, col) - if not always_create: - for alias in self.join_map.get(t_ident, ()): - if alias not in exclusions: - if lhs_table and not self.alias_refcount[self.alias_map[alias][LHS_ALIAS]]: - # The LHS of this join tuple is no longer part of the - # query, so skip this possibility. - continue - if self.alias_map[alias][LHS_ALIAS] != lhs: - continue - self.ref_alias(alias) - if promote: - self.promote_alias(alias) - return alias - - # No reuse is possible, so we need a new alias. - alias, _ = self.table_alias(table, True) - if not lhs: - # Not all tables need to be joined to anything. No join type - # means the later columns are ignored. - join_type = None - elif promote or outer_if_first: - join_type = self.LOUTER - else: - join_type = self.INNER - join = (table, alias, join_type, lhs, lhs_col, col, nullable) - self.alias_map[alias] = join - if t_ident in self.join_map: - self.join_map[t_ident] += (alias,) - else: - self.join_map[t_ident] = (alias,) - self.rev_join_map[alias] = t_ident - return alias - - def setup_inherited_models(self): - """ - If the model that is the basis for this QuerySet inherits other models, - we need to ensure that those other models have their tables included in - the query. - - We do this as a separate step so that subclasses know which - tables are going to be active in the query, without needing to compute - all the select columns (this method is called from pre_sql_setup(), - whereas column determination is a later part, and side-effect, of - as_sql()). - """ - opts = self.model._meta - root_alias = self.tables[0] - seen = {None: root_alias} - - # Skip all proxy to the root proxied model - proxied_model = get_proxied_model(opts) - - for field, model in opts.get_fields_with_model(): - if model not in seen: - if model is proxied_model: - seen[model] = root_alias - else: - link_field = opts.get_ancestor_link(model) - seen[model] = self.join((root_alias, model._meta.db_table, - link_field.column, model._meta.pk.column)) - self.included_inherited_models = seen - - def remove_inherited_models(self): - """ - Undoes the effects of setup_inherited_models(). Should be called - whenever select columns (self.select) are set explicitly. - """ - for key, alias in self.included_inherited_models.items(): - if key: - self.unref_alias(alias) - self.included_inherited_models = {} - - - def add_aggregate(self, aggregate, model, alias, is_summary): - """ - Adds a single aggregate expression to the Query - """ - opts = model._meta - field_list = aggregate.lookup.split(LOOKUP_SEP) - if len(field_list) == 1 and aggregate.lookup in self.aggregates: - # Aggregate is over an annotation - field_name = field_list[0] - col = field_name - source = self.aggregates[field_name] - if not is_summary: - raise FieldError("Cannot compute %s('%s'): '%s' is an aggregate" % ( - aggregate.name, field_name, field_name)) - elif ((len(field_list) > 1) or - (field_list[0] not in [i.name for i in opts.fields]) or - self.group_by is None or - not is_summary): - # If: - # - the field descriptor has more than one part (foo__bar), or - # - the field descriptor is referencing an m2m/m2o field, or - # - this is a reference to a model field (possibly inherited), or - # - this is an annotation over a model field - # then we need to explore the joins that are required. - - field, source, opts, join_list, last, _ = self.setup_joins( - field_list, opts, self.get_initial_alias(), False) - - # Process the join chain to see if it can be trimmed - col, _, join_list = self.trim_joins(source, join_list, last, False) - - # If the aggregate references a model or field that requires a join, - # those joins must be LEFT OUTER - empty join rows must be returned - # in order for zeros to be returned for those aggregates. - for column_alias in join_list: - self.promote_alias(column_alias, unconditional=True) - - col = (join_list[-1], col) - else: - # The simplest cases. No joins required - - # just reference the provided column alias. - field_name = field_list[0] - source = opts.get_field(field_name) - col = field_name - - # Add the aggregate to the query - aggregate.add_to_query(self, alias, col=col, source=source, is_summary=is_summary) - - def add_filter(self, filter_expr, connector=AND, negate=False, trim=False, - can_reuse=None, process_extras=True): - """ - Add a single filter to the query. The 'filter_expr' is a pair: - (filter_string, value). E.g. ('name__contains', 'fred') - - If 'negate' is True, this is an exclude() filter. It's important to - note that this method does not negate anything in the where-clause - object when inserting the filter constraints. This is because negated - filters often require multiple calls to add_filter() and the negation - should only happen once. So the caller is responsible for this (the - caller will normally be add_q(), so that as an example). - - If 'trim' is True, we automatically trim the final join group (used - internally when constructing nested queries). - - If 'can_reuse' is a set, we are processing a component of a - multi-component filter (e.g. filter(Q1, Q2)). In this case, 'can_reuse' - will be a set of table aliases that can be reused in this filter, even - if we would otherwise force the creation of new aliases for a join - (needed for nested Q-filters). The set is updated by this method. - - If 'process_extras' is set, any extra filters returned from the table - joining process will be processed. This parameter is set to False - during the processing of extra filters to avoid infinite recursion. - """ - arg, value = filter_expr - parts = arg.split(LOOKUP_SEP) - if not parts: - raise FieldError("Cannot parse keyword query %r" % arg) - - # Work out the lookup type and remove it from 'parts', if necessary. - if len(parts) == 1 or parts[-1] not in self.query_terms: - lookup_type = 'exact' - else: - lookup_type = parts.pop() - - # By default, this is a WHERE clause. If an aggregate is referenced - # in the value, the filter will be promoted to a HAVING - having_clause = False - - # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all - # uses of None as a query value. - if value is None: - if lookup_type != 'exact': - raise ValueError("Cannot use None as a query value") - lookup_type = 'isnull' - value = True - elif callable(value): - value = value() - elif hasattr(value, 'evaluate'): - # If value is a query expression, evaluate it - value = SQLEvaluator(value, self) - having_clause = value.contains_aggregate - - for alias, aggregate in self.aggregates.items(): - if alias == parts[0]: - entry = self.where_class() - entry.add((aggregate, lookup_type, value), AND) - if negate: - entry.negate() - self.having.add(entry, AND) - return - - opts = self.get_meta() - alias = self.get_initial_alias() - allow_many = trim or not negate - - try: - field, target, opts, join_list, last, extra_filters = self.setup_joins( - parts, opts, alias, True, allow_many, can_reuse=can_reuse, - negate=negate, process_extras=process_extras) - except MultiJoin, e: - self.split_exclude(filter_expr, LOOKUP_SEP.join(parts[:e.level]), - can_reuse) - return - - if (lookup_type == 'isnull' and value is True and not negate and - len(join_list) > 1): - # If the comparison is against NULL, we may need to use some left - # outer joins when creating the join chain. This is only done when - # needed, as it's less efficient at the database level. - self.promote_alias_chain(join_list) - - # Process the join list to see if we can remove any inner joins from - # the far end (fewer tables in a query is better). - col, alias, join_list = self.trim_joins(target, join_list, last, trim) - - if connector == OR: - # Some joins may need to be promoted when adding a new filter to a - # disjunction. We walk the list of new joins and where it diverges - # from any previous joins (ref count is 1 in the table list), we - # make the new additions (and any existing ones not used in the new - # join list) an outer join. - join_it = iter(join_list) - table_it = iter(self.tables) - join_it.next(), table_it.next() - table_promote = False - join_promote = False - for join in join_it: - table = table_it.next() - if join == table and self.alias_refcount[join] > 1: - continue - join_promote = self.promote_alias(join) - if table != join: - table_promote = self.promote_alias(table) - break - self.promote_alias_chain(join_it, join_promote) - self.promote_alias_chain(table_it, table_promote) - - - if having_clause: - if (alias, col) not in self.group_by: - self.group_by.append((alias, col)) - self.having.add((Constraint(alias, col, field), lookup_type, value), - connector) - else: - self.where.add((Constraint(alias, col, field), lookup_type, value), - connector) - - if negate: - self.promote_alias_chain(join_list) - if lookup_type != 'isnull': - if len(join_list) > 1: - for alias in join_list: - if self.alias_map[alias][JOIN_TYPE] == self.LOUTER: - j_col = self.alias_map[alias][RHS_JOIN_COL] - entry = self.where_class() - entry.add( - (Constraint(alias, j_col, None), 'isnull', True), - AND - ) - entry.negate() - self.where.add(entry, AND) - break - if not (lookup_type == 'in' - and not hasattr(value, 'as_sql') - and not hasattr(value, '_as_sql') - and not value) and field.null: - # Leaky abstraction artifact: We have to specifically - # exclude the "foo__in=[]" case from this handling, because - # it's short-circuited in the Where class. - # We also need to handle the case where a subquery is provided - self.where.add((Constraint(alias, col, None), 'isnull', False), AND) - - if can_reuse is not None: - can_reuse.update(join_list) - if process_extras: - for filter in extra_filters: - self.add_filter(filter, negate=negate, can_reuse=can_reuse, - process_extras=False) - - def add_q(self, q_object, used_aliases=None): - """ - Adds a Q-object to the current filter. - - Can also be used to add anything that has an 'add_to_query()' method. - """ - if used_aliases is None: - used_aliases = self.used_aliases - if hasattr(q_object, 'add_to_query'): - # Complex custom objects are responsible for adding themselves. - q_object.add_to_query(self, used_aliases) - else: - if self.where and q_object.connector != AND and len(q_object) > 1: - self.where.start_subtree(AND) - subtree = True - else: - subtree = False - connector = AND - for child in q_object.children: - if connector == OR: - refcounts_before = self.alias_refcount.copy() - self.where.start_subtree(connector) - if isinstance(child, Node): - self.add_q(child, used_aliases) - else: - self.add_filter(child, connector, q_object.negated, - can_reuse=used_aliases) - self.where.end_subtree() - if connector == OR: - # Aliases that were newly added or not used at all need to - # be promoted to outer joins if they are nullable relations. - # (they shouldn't turn the whole conditional into the empty - # set just because they don't match anything). - self.promote_unused_aliases(refcounts_before, used_aliases) - connector = q_object.connector - if q_object.negated: - self.where.negate() - if subtree: - self.where.end_subtree() - if self.filter_is_sticky: - self.used_aliases = used_aliases - - def setup_joins(self, names, opts, alias, dupe_multis, allow_many=True, - allow_explicit_fk=False, can_reuse=None, negate=False, - process_extras=True): - """ - Compute the necessary table joins for the passage through the fields - given in 'names'. 'opts' is the Options class for the current model - (which gives the table we are joining to), 'alias' is the alias for the - table we are joining to. If dupe_multis is True, any many-to-many or - many-to-one joins will always create a new alias (necessary for - disjunctive filters). If can_reuse is not None, it's a list of aliases - that can be reused in these joins (nothing else can be reused in this - case). Finally, 'negate' is used in the same sense as for add_filter() - -- it indicates an exclude() filter, or something similar. It is only - passed in here so that it can be passed to a field's extra_filter() for - customised behaviour. - - Returns the final field involved in the join, the target database - column (used for any 'where' constraint), the final 'opts' value and the - list of tables joined. - """ - joins = [alias] - last = [0] - dupe_set = set() - exclusions = set() - extra_filters = [] - for pos, name in enumerate(names): - try: - exclusions.add(int_alias) - except NameError: - pass - exclusions.add(alias) - last.append(len(joins)) - if name == 'pk': - name = opts.pk.name - try: - field, model, direct, m2m = opts.get_field_by_name(name) - except FieldDoesNotExist: - for f in opts.fields: - if allow_explicit_fk and name == f.attname: - # XXX: A hack to allow foo_id to work in values() for - # backwards compatibility purposes. If we dropped that - # feature, this could be removed. - field, model, direct, m2m = opts.get_field_by_name(f.name) - break - else: - names = opts.get_all_field_names() + self.aggregate_select.keys() - raise FieldError("Cannot resolve keyword %r into field. " - "Choices are: %s" % (name, ", ".join(names))) - - if not allow_many and (m2m or not direct): - for alias in joins: - self.unref_alias(alias) - raise MultiJoin(pos + 1) - if model: - # The field lives on a base class of the current model. - # Skip the chain of proxy to the concrete proxied model - proxied_model = get_proxied_model(opts) - - for int_model in opts.get_base_chain(model): - if int_model is proxied_model: - opts = int_model._meta - else: - lhs_col = opts.parents[int_model].column - dedupe = lhs_col in opts.duplicate_targets - if dedupe: - exclusions.update(self.dupe_avoidance.get( - (id(opts), lhs_col), ())) - dupe_set.add((opts, lhs_col)) - opts = int_model._meta - alias = self.join((alias, opts.db_table, lhs_col, - opts.pk.column), exclusions=exclusions) - joins.append(alias) - exclusions.add(alias) - for (dupe_opts, dupe_col) in dupe_set: - self.update_dupe_avoidance(dupe_opts, dupe_col, - alias) - cached_data = opts._join_cache.get(name) - orig_opts = opts - dupe_col = direct and field.column or field.field.column - dedupe = dupe_col in opts.duplicate_targets - if dupe_set or dedupe: - if dedupe: - dupe_set.add((opts, dupe_col)) - exclusions.update(self.dupe_avoidance.get((id(opts), dupe_col), - ())) - - if process_extras and hasattr(field, 'extra_filters'): - extra_filters.extend(field.extra_filters(names, pos, negate)) - if direct: - if m2m: - # Many-to-many field defined on the current model. - if cached_data: - (table1, from_col1, to_col1, table2, from_col2, - to_col2, opts, target) = cached_data - else: - table1 = field.m2m_db_table() - from_col1 = opts.pk.column - to_col1 = field.m2m_column_name() - opts = field.rel.to._meta - table2 = opts.db_table - from_col2 = field.m2m_reverse_name() - to_col2 = opts.pk.column - target = opts.pk - orig_opts._join_cache[name] = (table1, from_col1, - to_col1, table2, from_col2, to_col2, opts, - target) - - int_alias = self.join((alias, table1, from_col1, to_col1), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) - if int_alias == table2 and from_col2 == to_col2: - joins.append(int_alias) - alias = int_alias - else: - alias = self.join( - (int_alias, table2, from_col2, to_col2), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) - joins.extend([int_alias, alias]) - elif field.rel: - # One-to-one or many-to-one field - if cached_data: - (table, from_col, to_col, opts, target) = cached_data - else: - opts = field.rel.to._meta - target = field.rel.get_related_field() - table = opts.db_table - from_col = field.column - to_col = target.column - orig_opts._join_cache[name] = (table, from_col, to_col, - opts, target) - - alias = self.join((alias, table, from_col, to_col), - exclusions=exclusions, nullable=field.null) - joins.append(alias) - else: - # Non-relation fields. - target = field - break - else: - orig_field = field - field = field.field - if m2m: - # Many-to-many field defined on the target model. - if cached_data: - (table1, from_col1, to_col1, table2, from_col2, - to_col2, opts, target) = cached_data - else: - table1 = field.m2m_db_table() - from_col1 = opts.pk.column - to_col1 = field.m2m_reverse_name() - opts = orig_field.opts - table2 = opts.db_table - from_col2 = field.m2m_column_name() - to_col2 = opts.pk.column - target = opts.pk - orig_opts._join_cache[name] = (table1, from_col1, - to_col1, table2, from_col2, to_col2, opts, - target) - - int_alias = self.join((alias, table1, from_col1, to_col1), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) - alias = self.join((int_alias, table2, from_col2, to_col2), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) - joins.extend([int_alias, alias]) - else: - # One-to-many field (ForeignKey defined on the target model) - if cached_data: - (table, from_col, to_col, opts, target) = cached_data - else: - local_field = opts.get_field_by_name( - field.rel.field_name)[0] - opts = orig_field.opts - table = opts.db_table - from_col = local_field.column - to_col = field.column - target = opts.pk - orig_opts._join_cache[name] = (table, from_col, to_col, - opts, target) - - alias = self.join((alias, table, from_col, to_col), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) - joins.append(alias) - - for (dupe_opts, dupe_col) in dupe_set: - try: - self.update_dupe_avoidance(dupe_opts, dupe_col, int_alias) - except NameError: - self.update_dupe_avoidance(dupe_opts, dupe_col, alias) - - if pos != len(names) - 1: - if pos == len(names) - 2: - raise FieldError("Join on field %r not permitted. Did you misspell %r for the lookup type?" % (name, names[pos + 1])) - else: - raise FieldError("Join on field %r not permitted." % name) - - return field, target, opts, joins, last, extra_filters - - def trim_joins(self, target, join_list, last, trim): - """ - Sometimes joins at the end of a multi-table sequence can be trimmed. If - the final join is against the same column as we are comparing against, - and is an inner join, we can go back one step in a join chain and - compare against the LHS of the join instead (and then repeat the - optimization). The result, potentially, involves less table joins. - - The 'target' parameter is the final field being joined to, 'join_list' - is the full list of join aliases. - - The 'last' list contains offsets into 'join_list', corresponding to - each component of the filter. Many-to-many relations, for example, add - two tables to the join list and we want to deal with both tables the - same way, so 'last' has an entry for the first of the two tables and - then the table immediately after the second table, in that case. - - The 'trim' parameter forces the final piece of the join list to be - trimmed before anything. See the documentation of add_filter() for - details about this. - - Returns the final active column and table alias and the new active - join_list. - """ - final = len(join_list) - penultimate = last.pop() - if penultimate == final: - penultimate = last.pop() - if trim and len(join_list) > 1: - extra = join_list[penultimate:] - join_list = join_list[:penultimate] - final = penultimate - penultimate = last.pop() - col = self.alias_map[extra[0]][LHS_JOIN_COL] - for alias in extra: - self.unref_alias(alias) - else: - col = target.column - alias = join_list[-1] - while final > 1: - join = self.alias_map[alias] - if col != join[RHS_JOIN_COL] or join[JOIN_TYPE] != self.INNER: - break - self.unref_alias(alias) - alias = join[LHS_ALIAS] - col = join[LHS_JOIN_COL] - join_list = join_list[:-1] - final -= 1 - if final == penultimate: - penultimate = last.pop() - return col, alias, join_list - - def update_dupe_avoidance(self, opts, col, alias): - """ - For a column that is one of multiple pointing to the same table, update - the internal data structures to note that this alias shouldn't be used - for those other columns. - """ - ident = id(opts) - for name in opts.duplicate_targets[col]: - try: - self.dupe_avoidance[ident, name].add(alias) - except KeyError: - self.dupe_avoidance[ident, name] = set([alias]) - - def split_exclude(self, filter_expr, prefix, can_reuse): - """ - When doing an exclude against any kind of N-to-many relation, we need - to use a subquery. This method constructs the nested query, given the - original exclude filter (filter_expr) and the portion up to the first - N-to-many relation field. - """ - query = Query(self.model) - query.add_filter(filter_expr, can_reuse=can_reuse) - query.bump_prefix() - query.clear_ordering(True) - query.set_start(prefix) - self.add_filter(('%s__in' % prefix, query), negate=True, trim=True, - can_reuse=can_reuse) - - # If there's more than one join in the inner query (before any initial - # bits were trimmed -- which means the last active table is more than - # two places into the alias list), we need to also handle the - # possibility that the earlier joins don't match anything by adding a - # comparison to NULL (e.g. in - # Tag.objects.exclude(parent__parent__name='t1'), a tag with no parent - # would otherwise be overlooked). - active_positions = [pos for (pos, count) in - enumerate(query.alias_refcount.itervalues()) if count] - if active_positions[-1] > 1: - self.add_filter(('%s__isnull' % prefix, False), negate=True, - trim=True, can_reuse=can_reuse) - - def set_limits(self, low=None, high=None): - """ - Adjusts the limits on the rows retrieved. We use low/high to set these, - as it makes it more Pythonic to read and write. When the SQL query is - created, they are converted to the appropriate offset and limit values. - - Any limits passed in here are applied relative to the existing - constraints. So low is added to the current low value and both will be - clamped to any existing high value. - """ - if high is not None: - if self.high_mark is not None: - self.high_mark = min(self.high_mark, self.low_mark + high) - else: - self.high_mark = self.low_mark + high - if low is not None: - if self.high_mark is not None: - self.low_mark = min(self.high_mark, self.low_mark + low) - else: - self.low_mark = self.low_mark + low - - def clear_limits(self): - """ - Clears any existing limits. - """ - self.low_mark, self.high_mark = 0, None - - def can_filter(self): - """ - Returns True if adding filters to this instance is still possible. - - Typically, this means no limits or offsets have been put on the results. - """ - return not self.low_mark and self.high_mark is None - - def clear_select_fields(self): - """ - Clears the list of fields to select (but not extra_select columns). - Some queryset types completely replace any existing list of select - columns. - """ - self.select = [] - self.select_fields = [] - - def add_fields(self, field_names, allow_m2m=True): - """ - Adds the given (model) fields to the select set. The field names are - added in the order specified. - """ - alias = self.get_initial_alias() - opts = self.get_meta() - - try: - for name in field_names: - field, target, u2, joins, u3, u4 = self.setup_joins( - name.split(LOOKUP_SEP), opts, alias, False, allow_m2m, - True) - final_alias = joins[-1] - col = target.column - if len(joins) > 1: - join = self.alias_map[final_alias] - if col == join[RHS_JOIN_COL]: - self.unref_alias(final_alias) - final_alias = join[LHS_ALIAS] - col = join[LHS_JOIN_COL] - joins = joins[:-1] - self.promote_alias_chain(joins[1:]) - self.select.append((final_alias, col)) - self.select_fields.append(field) - except MultiJoin: - raise FieldError("Invalid field name: '%s'" % name) - except FieldError: - names = opts.get_all_field_names() + self.extra.keys() + self.aggregate_select.keys() - names.sort() - raise FieldError("Cannot resolve keyword %r into field. " - "Choices are: %s" % (name, ", ".join(names))) - self.remove_inherited_models() - - def add_ordering(self, *ordering): - """ - Adds items from the 'ordering' sequence to the query's "order by" - clause. These items are either field names (not column names) -- - possibly with a direction prefix ('-' or '?') -- or ordinals, - corresponding to column positions in the 'select' list. - - If 'ordering' is empty, all ordering is cleared from the query. - """ - errors = [] - for item in ordering: - if not ORDER_PATTERN.match(item): - errors.append(item) - if errors: - raise FieldError('Invalid order_by arguments: %s' % errors) - if ordering: - self.order_by.extend(ordering) - else: - self.default_ordering = False - - def clear_ordering(self, force_empty=False): - """ - Removes any ordering settings. If 'force_empty' is True, there will be - no ordering in the resulting query (not even the model's default). - """ - self.order_by = [] - self.extra_order_by = () - if force_empty: - self.default_ordering = False - - def set_group_by(self): - """ - Expands the GROUP BY clause required by the query. - - This will usually be the set of all non-aggregate fields in the - return data. If the database backend supports grouping by the - primary key, and the query would be equivalent, the optimization - will be made automatically. - """ - self.group_by = [] - - for sel in self.select: - self.group_by.append(sel) - - def add_count_column(self): - """ - Converts the query to do count(...) or count(distinct(pk)) in order to - get its size. - """ - if not self.distinct: - if not self.select: - count = self.aggregates_module.Count('*', is_summary=True) - else: - assert len(self.select) == 1, \ - "Cannot add count col with multiple cols in 'select': %r" % self.select - count = self.aggregates_module.Count(self.select[0]) - else: - opts = self.model._meta - if not self.select: - count = self.aggregates_module.Count((self.join((None, opts.db_table, None, None)), opts.pk.column), - is_summary=True, distinct=True) - else: - # Because of SQL portability issues, multi-column, distinct - # counts need a sub-query -- see get_count() for details. - assert len(self.select) == 1, \ - "Cannot add count col with multiple cols in 'select'." - - count = self.aggregates_module.Count(self.select[0], distinct=True) - # Distinct handling is done in Count(), so don't do it at this - # level. - self.distinct = False - - # Set only aggregate to be the count column. - # Clear out the select cache to reflect the new unmasked aggregates. - self.aggregates = {None: count} - self.set_aggregate_mask(None) - self.group_by = None - - def add_select_related(self, fields): - """ - Sets up the select_related data structure so that we only select - certain related models (as opposed to all models, when - self.select_related=True). - """ - field_dict = {} - for field in fields: - d = field_dict - for part in field.split(LOOKUP_SEP): - d = d.setdefault(part, {}) - self.select_related = field_dict - self.related_select_cols = [] - self.related_select_fields = [] - - def add_extra(self, select, select_params, where, params, tables, order_by): - """ - Adds data to the various extra_* attributes for user-created additions - to the query. - """ - if select: - # We need to pair any placeholder markers in the 'select' - # dictionary with their parameters in 'select_params' so that - # subsequent updates to the select dictionary also adjust the - # parameters appropriately. - select_pairs = SortedDict() - if select_params: - param_iter = iter(select_params) - else: - param_iter = iter([]) - for name, entry in select.items(): - entry = force_unicode(entry) - entry_params = [] - pos = entry.find("%s") - while pos != -1: - entry_params.append(param_iter.next()) - pos = entry.find("%s", pos + 2) - select_pairs[name] = (entry, entry_params) - # This is order preserving, since self.extra_select is a SortedDict. - self.extra.update(select_pairs) - if where or params: - self.where.add(ExtraWhere(where, params), AND) - if tables: - self.extra_tables += tuple(tables) - if order_by: - self.extra_order_by = order_by - - def clear_deferred_loading(self): - """ - Remove any fields from the deferred loading set. - """ - self.deferred_loading = (set(), True) - - def add_deferred_loading(self, field_names): - """ - Add the given list of model field names to the set of fields to - exclude from loading from the database when automatic column selection - is done. The new field names are added to any existing field names that - are deferred (or removed from any existing field names that are marked - as the only ones for immediate loading). - """ - # Fields on related models are stored in the literal double-underscore - # format, so that we can use a set datastructure. We do the foo__bar - # splitting and handling when computing the SQL colum names (as part of - # get_columns()). - existing, defer = self.deferred_loading - if defer: - # Add to existing deferred names. - self.deferred_loading = existing.union(field_names), True - else: - # Remove names from the set of any existing "immediate load" names. - self.deferred_loading = existing.difference(field_names), False - - def add_immediate_loading(self, field_names): - """ - Add the given list of model field names to the set of fields to - retrieve when the SQL is executed ("immediate loading" fields). The - field names replace any existing immediate loading field names. If - there are field names already specified for deferred loading, those - names are removed from the new field_names before storing the new names - for immediate loading. (That is, immediate loading overrides any - existing immediate values, but respects existing deferrals.) - """ - existing, defer = self.deferred_loading - if defer: - # Remove any existing deferred names from the current set before - # setting the new names. - self.deferred_loading = set(field_names).difference(existing), False - else: - # Replace any existing "immediate load" field names. - self.deferred_loading = set(field_names), False - - def get_loaded_field_names(self): - """ - If any fields are marked to be deferred, returns a dictionary mapping - models to a set of names in those fields that will be loaded. If a - model is not in the returned dictionary, none of it's fields are - deferred. - - If no fields are marked for deferral, returns an empty dictionary. - """ - collection = {} - self.deferred_to_data(collection, self.get_loaded_field_names_cb) - return collection - - def get_loaded_field_names_cb(self, target, model, fields): - """ - Callback used by get_deferred_field_names(). - """ - target[model] = set([f.name for f in fields]) - - def set_aggregate_mask(self, names): - "Set the mask of aggregates that will actually be returned by the SELECT" - if names is None: - self.aggregate_select_mask = None - else: - self.aggregate_select_mask = set(names) - self._aggregate_select_cache = None - - def set_extra_mask(self, names): - """ - Set the mask of extra select items that will be returned by SELECT, - we don't actually remove them from the Query since they might be used - later - """ - if names is None: - self.extra_select_mask = None - else: - self.extra_select_mask = set(names) - self._extra_select_cache = None - - def _aggregate_select(self): - """The SortedDict of aggregate columns that are not masked, and should - be used in the SELECT clause. - - This result is cached for optimization purposes. - """ - if self._aggregate_select_cache is not None: - return self._aggregate_select_cache - elif self.aggregate_select_mask is not None: - self._aggregate_select_cache = SortedDict([ - (k,v) for k,v in self.aggregates.items() - if k in self.aggregate_select_mask - ]) - return self._aggregate_select_cache - else: - return self.aggregates - aggregate_select = property(_aggregate_select) - - def _extra_select(self): - if self._extra_select_cache is not None: - return self._extra_select_cache - elif self.extra_select_mask is not None: - self._extra_select_cache = SortedDict([ - (k,v) for k,v in self.extra.items() - if k in self.extra_select_mask - ]) - return self._extra_select_cache - else: - return self.extra - extra_select = property(_extra_select) - - def set_start(self, start): - """ - Sets the table from which to start joining. The start position is - specified by the related attribute from the base model. This will - automatically set to the select column to be the column linked from the - previous table. - - This method is primarily for internal use and the error checking isn't - as friendly as add_filter(). Mostly useful for querying directly - against the join table of many-to-many relation in a subquery. - """ - opts = self.model._meta - alias = self.get_initial_alias() - field, col, opts, joins, last, extra = self.setup_joins( - start.split(LOOKUP_SEP), opts, alias, False) - select_col = self.alias_map[joins[1]][LHS_JOIN_COL] - select_alias = alias - - # The call to setup_joins added an extra reference to everything in - # joins. Reverse that. - for alias in joins: - self.unref_alias(alias) - - # We might be able to trim some joins from the front of this query, - # providing that we only traverse "always equal" connections (i.e. rhs - # is *always* the same value as lhs). - for alias in joins[1:]: - join_info = self.alias_map[alias] - if (join_info[LHS_JOIN_COL] != select_col - or join_info[JOIN_TYPE] != self.INNER): - break - self.unref_alias(select_alias) - select_alias = join_info[RHS_ALIAS] - select_col = join_info[RHS_JOIN_COL] - self.select = [(select_alias, select_col)] - self.remove_inherited_models() - - -def get_order_dir(field, default='ASC'): - """ - Returns the field name and direction for an order specification. For - example, '-foo' is returned as ('foo', 'DESC'). - - The 'default' param is used to indicate which way no prefix (or a '+' - prefix) should sort. The '-' prefix always sorts the opposite way. - """ - dirn = ORDER_DIR[default] - if field[0] == '-': - return field[1:], dirn[1] - return field, dirn[0] - - -def setup_join_cache(sender, **kwargs): - """ - The information needed to join between model fields is something that is - invariant over the life of the model, so we cache it in the model's Options - class, rather than recomputing it all the time. - - This method initialises the (empty) cache when the model is created. - """ - sender._meta._join_cache = {} - -signals.class_prepared.connect(setup_join_cache) - -def add_to_dict(data, key, value): - """ - A helper function to add "value" to the set of values for "key", whether or - not "key" already exists. - """ - if key in data: - data[key].add(value) - else: - data[key] = set([value]) - -def get_proxied_model(opts): - int_opts = opts - proxied_model = None - while int_opts.proxy: - proxied_model = int_opts.proxy_for_model - int_opts = proxied_model._meta - return proxied_model diff --git a/parts/django/django/db/models/sql/subqueries.py b/parts/django/django/db/models/sql/subqueries.py deleted file mode 100644 index a0bdc94..0000000 --- a/parts/django/django/db/models/sql/subqueries.py +++ /dev/null @@ -1,212 +0,0 @@ -""" -Query subclasses which provide extra functionality beyond simple data retrieval. -""" - -from django.core.exceptions import FieldError -from django.db import connections -from django.db.models.sql.constants import * -from django.db.models.sql.datastructures import Date -from django.db.models.sql.expressions import SQLEvaluator -from django.db.models.sql.query import Query -from django.db.models.sql.where import AND, Constraint - - -__all__ = ['DeleteQuery', 'UpdateQuery', 'InsertQuery', 'DateQuery', - 'AggregateQuery'] - -class DeleteQuery(Query): - """ - Delete queries are done through this class, since they are more constrained - than general queries. - """ - - compiler = 'SQLDeleteCompiler' - - def do_query(self, table, where, using): - self.tables = [table] - self.where = where - self.get_compiler(using).execute_sql(None) - - def delete_batch(self, pk_list, using): - """ - Set up and execute delete queries for all the objects in pk_list. - - More than one physical query may be executed if there are a - lot of values in pk_list. - """ - for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): - where = self.where_class() - field = self.model._meta.pk - where.add((Constraint(None, field.column, field), 'in', - pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]), AND) - self.do_query(self.model._meta.db_table, where, using=using) - -class UpdateQuery(Query): - """ - Represents an "update" SQL query. - """ - - compiler = 'SQLUpdateCompiler' - - def __init__(self, *args, **kwargs): - super(UpdateQuery, self).__init__(*args, **kwargs) - self._setup_query() - - def _setup_query(self): - """ - Runs on initialization and after cloning. Any attributes that would - normally be set in __init__ should go in here, instead, so that they - are also set up after a clone() call. - """ - self.values = [] - self.related_ids = None - if not hasattr(self, 'related_updates'): - self.related_updates = {} - - def clone(self, klass=None, **kwargs): - return super(UpdateQuery, self).clone(klass, - related_updates=self.related_updates.copy(), **kwargs) - - - def clear_related(self, related_field, pk_list, using): - """ - Set up and execute an update query that clears related entries for the - keys in pk_list. - - This is used by the QuerySet.delete_objects() method. - """ - for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): - self.where = self.where_class() - f = self.model._meta.pk - self.where.add((Constraint(None, f.column, f), 'in', - pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]), - AND) - self.values = [(related_field, None, None)] - self.get_compiler(using).execute_sql(None) - - def add_update_values(self, values): - """ - Convert a dictionary of field name to value mappings into an update - query. This is the entry point for the public update() method on - querysets. - """ - values_seq = [] - for name, val in values.iteritems(): - field, model, direct, m2m = self.model._meta.get_field_by_name(name) - if not direct or m2m: - raise FieldError('Cannot update model field %r (only non-relations and foreign keys permitted).' % field) - if model: - self.add_related_update(model, field, val) - continue - values_seq.append((field, model, val)) - return self.add_update_fields(values_seq) - - def add_update_fields(self, values_seq): - """ - Turn a sequence of (field, model, value) triples into an update query. - Used by add_update_values() as well as the "fast" update path when - saving models. - """ - self.values.extend(values_seq) - - def add_related_update(self, model, field, value): - """ - Adds (name, value) to an update query for an ancestor model. - - Updates are coalesced so that we only run one update query per ancestor. - """ - try: - self.related_updates[model].append((field, None, value)) - except KeyError: - self.related_updates[model] = [(field, None, value)] - - def get_related_updates(self): - """ - Returns a list of query objects: one for each update required to an - ancestor model. Each query will have the same filtering conditions as - the current query but will only update a single table. - """ - if not self.related_updates: - return [] - result = [] - for model, values in self.related_updates.iteritems(): - query = UpdateQuery(model) - query.values = values - if self.related_ids is not None: - query.add_filter(('pk__in', self.related_ids)) - result.append(query) - return result - -class InsertQuery(Query): - compiler = 'SQLInsertCompiler' - - def __init__(self, *args, **kwargs): - super(InsertQuery, self).__init__(*args, **kwargs) - self.columns = [] - self.values = [] - self.params = () - - def clone(self, klass=None, **kwargs): - extras = { - 'columns': self.columns[:], - 'values': self.values[:], - 'params': self.params - } - extras.update(kwargs) - return super(InsertQuery, self).clone(klass, **extras) - - def insert_values(self, insert_values, raw_values=False): - """ - Set up the insert query from the 'insert_values' dictionary. The - dictionary gives the model field names and their target values. - - If 'raw_values' is True, the values in the 'insert_values' dictionary - are inserted directly into the query, rather than passed as SQL - parameters. This provides a way to insert NULL and DEFAULT keywords - into the query, for example. - """ - placeholders, values = [], [] - for field, val in insert_values: - placeholders.append((field, val)) - self.columns.append(field.column) - values.append(val) - if raw_values: - self.values.extend([(None, v) for v in values]) - else: - self.params += tuple(values) - self.values.extend(placeholders) - -class DateQuery(Query): - """ - A DateQuery is a normal query, except that it specifically selects a single - date field. This requires some special handling when converting the results - back to Python objects, so we put it in a separate class. - """ - - compiler = 'SQLDateCompiler' - - def add_date_select(self, field, lookup_type, order='ASC'): - """ - Converts the query into a date extraction query. - """ - result = self.setup_joins([field.name], self.get_meta(), - self.get_initial_alias(), False) - alias = result[3][-1] - select = Date((alias, field.column), lookup_type) - self.select = [select] - self.select_fields = [None] - self.select_related = False # See #7097. - self.set_extra_mask([]) - self.distinct = True - self.order_by = order == 'ASC' and [1] or [-1] - -class AggregateQuery(Query): - """ - An AggregateQuery takes another query as a parameter to the FROM - clause and only selects the elements in the provided list. - """ - - compiler = 'SQLAggregateCompiler' - - def add_subquery(self, query, using): - self.subquery, self.sub_params = query.get_compiler(using).as_sql(with_col_aliases=True) diff --git a/parts/django/django/db/models/sql/where.py b/parts/django/django/db/models/sql/where.py deleted file mode 100644 index 2427a52..0000000 --- a/parts/django/django/db/models/sql/where.py +++ /dev/null @@ -1,345 +0,0 @@ -""" -Code to manage the creation and SQL rendering of 'where' constraints. -""" -import datetime -from itertools import repeat - -from django.utils import tree -from django.db.models.fields import Field -from django.db.models.query_utils import QueryWrapper -from datastructures import EmptyResultSet, FullResultSet - -# Connection types -AND = 'AND' -OR = 'OR' - -class EmptyShortCircuit(Exception): - """ - Internal exception used to indicate that a "matches nothing" node should be - added to the where-clause. - """ - pass - -class WhereNode(tree.Node): - """ - Used to represent the SQL where-clause. - - The class is tied to the Query class that created it (in order to create - the correct SQL). - - The children in this tree are usually either Q-like objects or lists of - [table_alias, field_name, db_type, lookup_type, value_annotation, - params]. However, a child could also be any class with as_sql() and - relabel_aliases() methods. - """ - default = AND - - def add(self, data, connector): - """ - Add a node to the where-tree. If the data is a list or tuple, it is - expected to be of the form (obj, lookup_type, value), where obj is - a Constraint object, and is then slightly munged before being stored - (to avoid storing any reference to field objects). Otherwise, the 'data' - is stored unchanged and can be any class with an 'as_sql()' method. - """ - if not isinstance(data, (list, tuple)): - super(WhereNode, self).add(data, connector) - return - - obj, lookup_type, value = data - if hasattr(value, '__iter__') and hasattr(value, 'next'): - # Consume any generators immediately, so that we can determine - # emptiness and transform any non-empty values correctly. - value = list(value) - - # The "annotation" parameter is used to pass auxilliary information - # about the value(s) to the query construction. Specifically, datetime - # and empty values need special handling. Other types could be used - # here in the future (using Python types is suggested for consistency). - if isinstance(value, datetime.datetime): - annotation = datetime.datetime - elif hasattr(value, 'value_annotation'): - annotation = value.value_annotation - else: - annotation = bool(value) - - if hasattr(obj, "prepare"): - value = obj.prepare(lookup_type, value) - super(WhereNode, self).add((obj, lookup_type, annotation, value), - connector) - return - - super(WhereNode, self).add((obj, lookup_type, annotation, value), - connector) - - def as_sql(self, qn, connection): - """ - Returns the SQL version of the where clause and the value to be - substituted in. Returns None, None if this node is empty. - - If 'node' is provided, that is the root of the SQL generation - (generally not needed except by the internal implementation for - recursion). - """ - if not self.children: - return None, [] - result = [] - result_params = [] - empty = True - for child in self.children: - try: - if hasattr(child, 'as_sql'): - sql, params = child.as_sql(qn=qn, connection=connection) - else: - # A leaf node in the tree. - sql, params = self.make_atom(child, qn, connection) - - except EmptyResultSet: - if self.connector == AND and not self.negated: - # We can bail out early in this particular case (only). - raise - elif self.negated: - empty = False - continue - except FullResultSet: - if self.connector == OR: - if self.negated: - empty = True - break - # We match everything. No need for any constraints. - return '', [] - if self.negated: - empty = True - continue - - empty = False - if sql: - result.append(sql) - result_params.extend(params) - if empty: - raise EmptyResultSet - - conn = ' %s ' % self.connector - sql_string = conn.join(result) - if sql_string: - if self.negated: - sql_string = 'NOT (%s)' % sql_string - elif len(self.children) != 1: - sql_string = '(%s)' % sql_string - return sql_string, result_params - - def make_atom(self, child, qn, connection): - """ - Turn a tuple (table_alias, column_name, db_type, lookup_type, - value_annot, params) into valid SQL. - - Returns the string for the SQL fragment and the parameters to use for - it. - """ - lvalue, lookup_type, value_annot, params_or_value = child - if hasattr(lvalue, 'process'): - try: - lvalue, params = lvalue.process(lookup_type, params_or_value, connection) - except EmptyShortCircuit: - raise EmptyResultSet - else: - params = Field().get_db_prep_lookup(lookup_type, params_or_value, - connection=connection, prepared=True) - if isinstance(lvalue, tuple): - # A direct database column lookup. - field_sql = self.sql_for_columns(lvalue, qn, connection) - else: - # A smart object with an as_sql() method. - field_sql = lvalue.as_sql(qn, connection) - - if value_annot is datetime.datetime: - cast_sql = connection.ops.datetime_cast_sql() - else: - cast_sql = '%s' - - if hasattr(params, 'as_sql'): - extra, params = params.as_sql(qn, connection) - cast_sql = '' - else: - extra = '' - - if (len(params) == 1 and params[0] == '' and lookup_type == 'exact' - and connection.features.interprets_empty_strings_as_nulls): - lookup_type = 'isnull' - value_annot = True - - if lookup_type in connection.operators: - format = "%s %%s %%s" % (connection.ops.lookup_cast(lookup_type),) - return (format % (field_sql, - connection.operators[lookup_type] % cast_sql, - extra), params) - - if lookup_type == 'in': - if not value_annot: - raise EmptyResultSet - if extra: - return ('%s IN %s' % (field_sql, extra), params) - max_in_list_size = connection.ops.max_in_list_size() - if max_in_list_size and len(params) > max_in_list_size: - # Break up the params list into an OR of manageable chunks. - in_clause_elements = ['('] - for offset in xrange(0, len(params), max_in_list_size): - if offset > 0: - in_clause_elements.append(' OR ') - in_clause_elements.append('%s IN (' % field_sql) - group_size = min(len(params) - offset, max_in_list_size) - param_group = ', '.join(repeat('%s', group_size)) - in_clause_elements.append(param_group) - in_clause_elements.append(')') - in_clause_elements.append(')') - return ''.join(in_clause_elements), params - else: - return ('%s IN (%s)' % (field_sql, - ', '.join(repeat('%s', len(params)))), - params) - elif lookup_type in ('range', 'year'): - return ('%s BETWEEN %%s and %%s' % field_sql, params) - elif lookup_type in ('month', 'day', 'week_day'): - return ('%s = %%s' % connection.ops.date_extract_sql(lookup_type, field_sql), - params) - elif lookup_type == 'isnull': - return ('%s IS %sNULL' % (field_sql, - (not value_annot and 'NOT ' or '')), ()) - elif lookup_type == 'search': - return (connection.ops.fulltext_search_sql(field_sql), params) - elif lookup_type in ('regex', 'iregex'): - return connection.ops.regex_lookup(lookup_type) % (field_sql, cast_sql), params - - raise TypeError('Invalid lookup_type: %r' % lookup_type) - - def sql_for_columns(self, data, qn, connection): - """ - Returns the SQL fragment used for the left-hand side of a column - constraint (for example, the "T1.foo" portion in the clause - "WHERE ... T1.foo = 6"). - """ - table_alias, name, db_type = data - if table_alias: - lhs = '%s.%s' % (qn(table_alias), qn(name)) - else: - lhs = qn(name) - return connection.ops.field_cast_sql(db_type) % lhs - - def relabel_aliases(self, change_map, node=None): - """ - Relabels the alias values of any children. 'change_map' is a dictionary - mapping old (current) alias values to the new values. - """ - if not node: - node = self - for pos, child in enumerate(node.children): - if hasattr(child, 'relabel_aliases'): - child.relabel_aliases(change_map) - elif isinstance(child, tree.Node): - self.relabel_aliases(change_map, child) - elif isinstance(child, (list, tuple)): - if isinstance(child[0], (list, tuple)): - elt = list(child[0]) - if elt[0] in change_map: - elt[0] = change_map[elt[0]] - node.children[pos] = (tuple(elt),) + child[1:] - else: - child[0].relabel_aliases(change_map) - - # Check if the query value also requires relabelling - if hasattr(child[3], 'relabel_aliases'): - child[3].relabel_aliases(change_map) - -class EverythingNode(object): - """ - A node that matches everything. - """ - - def as_sql(self, qn=None, connection=None): - raise FullResultSet - - def relabel_aliases(self, change_map, node=None): - return - -class NothingNode(object): - """ - A node that matches nothing. - """ - def as_sql(self, qn=None, connection=None): - raise EmptyResultSet - - def relabel_aliases(self, change_map, node=None): - return - -class ExtraWhere(object): - def __init__(self, sqls, params): - self.sqls = sqls - self.params = params - - def as_sql(self, qn=None, connection=None): - return " AND ".join(self.sqls), tuple(self.params or ()) - -class Constraint(object): - """ - An object that can be passed to WhereNode.add() and knows how to - pre-process itself prior to including in the WhereNode. - """ - def __init__(self, alias, col, field): - self.alias, self.col, self.field = alias, col, field - - def __getstate__(self): - """Save the state of the Constraint for pickling. - - Fields aren't necessarily pickleable, because they can have - callable default values. So, instead of pickling the field - store a reference so we can restore it manually - """ - obj_dict = self.__dict__.copy() - if self.field: - obj_dict['model'] = self.field.model - obj_dict['field_name'] = self.field.name - del obj_dict['field'] - return obj_dict - - def __setstate__(self, data): - """Restore the constraint """ - model = data.pop('model', None) - field_name = data.pop('field_name', None) - self.__dict__.update(data) - if model is not None: - self.field = model._meta.get_field(field_name) - else: - self.field = None - - def prepare(self, lookup_type, value): - if self.field: - return self.field.get_prep_lookup(lookup_type, value) - return value - - def process(self, lookup_type, value, connection): - """ - Returns a tuple of data suitable for inclusion in a WhereNode - instance. - """ - # Because of circular imports, we need to import this here. - from django.db.models.base import ObjectDoesNotExist - try: - if self.field: - params = self.field.get_db_prep_lookup(lookup_type, value, - connection=connection, prepared=True) - db_type = self.field.db_type(connection=connection) - else: - # This branch is used at times when we add a comparison to NULL - # (we don't really want to waste time looking up the associated - # field object at the calling location). - params = Field().get_db_prep_lookup(lookup_type, value, - connection=connection, prepared=True) - db_type = None - except ObjectDoesNotExist: - raise EmptyShortCircuit - - return (self.alias, self.col, db_type), params - - def relabel_aliases(self, change_map): - if self.alias in change_map: - self.alias = change_map[self.alias] |