casacore
TableParse.h
Go to the documentation of this file.
1 //# TableParse.h: Classes to hold results from table grammar parser
2 //# Copyright (C) 1994,1995,1997,1998,1999,2000,2001,2003
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef TABLES_TABLEPARSE_H
29 #define TABLES_TABLEPARSE_H
30 
31 //# Includes
32 #include <casacore/casa/aips.h>
33 #include <casacore/tables/Tables/Table.h>
34 #include <casacore/tables/Tables/TableDesc.h>
35 #include <casacore/tables/TaQL/ExprNode.h>
36 #include <casacore/tables/TaQL/TaQLResult.h>
37 #include <casacore/tables/TaQL/ExprGroup.h>
38 #include <casacore/casa/BasicSL/String.h>
39 #include <casacore/casa/Utilities/Sort.h>
40 #include <casacore/casa/Containers/Record.h>
41 #include <casacore/casa/Containers/Block.h>
42 #include <map>
43 #include <vector>
44 #include <limits>
45 
46 namespace casacore { //# NAMESPACE CASACORE - BEGIN
47 
48 //# Forward Declarations
49 class TableExprNodeSet;
50 class TableExprNodeSetElem;
51 class TableExprNodeIndex;
52 class TableColumn;
53 class AipsIO;
54 template<class T> class Vector;
55 
56 
57 // <summary>
58 // Class to hold values from table grammar parser
59 // </summary>
60 
61 // <use visibility=local>
62 
63 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="tTableGram">
64 // </reviewed>
65 
66 // <prerequisite>
67 //# Classes you should understand before using this one.
68 // </prerequisite>
69 
70 // <etymology>
71 // TableParse is the class used to parse a table command.
72 // </etymology>
73 
74 // <synopsis>
75 // TableParse is used by the parser of table select statements.
76 // The parser is written in Bison and Flex in files TableGram.y and .l.
77 // The statements in there use the routines in this file to act
78 // upon a reduced rule.
79 // Since multiple tables can be given (with a shorthand), the table
80 // names are stored in a container. The variable names can be qualified
81 // by the table name and will be looked up in the appropriate table.
82 //
83 // A select command is similar to SQL and can look like:
84 // SELECT columns FROM tab1 sh1, tab2 sh2, tab3 WHERE
85 // sh1.field == 3*sh1.field2 ... ORDERBY columns GIVING table
86 // This is described in more detail in TableGram.l.
87 //
88 // The class TableParse only contains information about a table
89 // used in the table command.
90 //
91 // Global functions are used to operate on the information.
92 // The main function is the global function tableCommand.
93 // It executes the given TaQL command and returns the resulting table.
94 // This is, in fact, the only function to be used by a user.
95 // </synopsis>
96 
97 // <motivation>
98 // It is necessary to be able to give a table select command in ASCII.
99 // This can be used in a CLI or in the table browser to get a subset
100 // of a table or to sort a table.
101 // </motivation>
102 
103 //# <todo asof="$DATE:$">
104 //# A List of bugs, limitations, extensions or planned refinements.
105 //# </todo>
106 
107 
109 {
110 
111 public:
112  // Default constructor for container class.
113  TableParse();
114 
115  // Copy constructor (copy semantics).
116  TableParse (const TableParse&);
117 
118  // Assignment (copy semantics).
120 
121  // Associate the table and the shorthand.
122  TableParse (const Table& table, const String& shorthand);
123 
124  // Test if shorthand matches.
125  Bool test (const String& shortHand) const;
126 
127  // Get the shorthand.
128  const String& shorthand() const;
129 
130  // Get table object.
131  const Table& table() const;
132 
133 private:
136 };
137 
138 
139 
140 // <synopsis>
141 // Parse and execute the given command.
142 // It will open (and close) all tables needed.
143 // It returns the resulting table.
144 // The command type (select or update) and the selected or updated
145 // column names can be returned.
146 // Zero or more temporary tables can be used in the command
147 // using the $nnn syntax.
148 // </synopsis>
149 // <group name=tableCommand>
150 TaQLResult tableCommand (const String& command);
151 
152 TaQLResult tableCommand (const String& command,
153  const Table& tempTable);
154 TaQLResult tableCommand (const String& command,
155  const std::vector<const Table*>& tempTables);
156 TaQLResult tableCommand (const String& command,
157  Vector<String>& columnNames);
158 TaQLResult tableCommand (const String& command,
159  Vector<String>& columnNames,
160  String& commandType);
161 TaQLResult tableCommand (const String& command,
162  const std::vector<const Table*>& tempTables,
163  Vector<String>& columnNames);
164 TaQLResult tableCommand (const String& command,
165  const std::vector<const Table*>& tempTables,
166  Vector<String>& columnNames,
167  String& commandType);
168 // </group>
169 
170 
171 
172 
173 // <summary>
174 // Helper class for sort keys in TableParse
175 // </summary>
176 
177 // <use visibility=local>
178 
179 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
180 // </reviewed>
181 
182 // <prerequisite>
183 //# Classes you should understand before using this one.
184 // <li> TableParse
185 // </prerequisite>
186 
187 // <etymology>
188 // TableParseSort holds a sort expression and order.
189 // </etymology>
190 
191 // <synopsis>
192 // A table command is parsed.
193 // An object of this class is used to hold the sort expression
194 // and sort order.
195 // </synopsis>
196 
197 
199 {
200 public:
201  // Construct from a given expression.
202  // The order is not given.
203  TableParseSort();
204 
205  // Construct from a given expression.
206  // The order is not given.
207  explicit TableParseSort (const TableExprNode&);
208 
209  // Construct from a given expression and for the given order.
211 
212  ~TableParseSort();
213 
214  // Get the expression node.
215  const TableExprNode& node() const;
216 
217  // Get the sort order.
218  Sort::Order order() const;
219 
220  // Is the order given?
221  Bool orderGiven() const;
222 
223 private:
224  // Check if the node results in a scalar and does not contain
225  // aggregate functions.
226  void checkNode() const;
227 
231 };
232 
233 
234 
235 
236 // <summary>
237 // Helper class for updates in TableParse
238 // </summary>
239 
240 // <use visibility=local>
241 
242 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
243 // </reviewed>
244 
245 // <prerequisite>
246 //# Classes you should understand before using this one.
247 // <li> TableParse
248 // </prerequisite>
249 
250 // <etymology>
251 // TableParseUpdate holds a column name, optional indices, optional mask,
252 // and an update expression.
253 // </etymology>
254 
255 // <synopsis>
256 // A table command is parsed.
257 // An object of this class is used to hold the column name, optional indices,
258 // and value expression for the UPDATE command.
259 // </synopsis>
260 
261 
263 {
264 public:
266  : indexPtr_p(0) {}
267 
268  // Construct from a column name and expression.
269  // By default it checks if no aggregate functions are used.
270  TableParseUpdate (const String& columnName,
271  const String& columnNameMask,
272  const TableExprNode&,
273  Bool checkAggr=True);
274 
275  // Construct from a column name, subscripts or mask, and expression.
276  // It checks if no aggregate functions are used.
277  TableParseUpdate (const String& columnName,
278  const String& columnNameMask,
279  const TableExprNodeSet& indices,
280  const TableExprNode&,
281  const TaQLStyle&);
282 
283  // Construct from a column name, subscripts and mask, and expression.
284  // It checks if no aggregate functions are used.
285  // It checks if one of the indices represents subscripts, the other a mask.
286  TableParseUpdate (const String& columnName,
287  const String& columnNameMask,
288  const TableExprNodeSet& indices1,
289  const TableExprNodeSet& indices2,
290  const TableExprNode&,
291  const TaQLStyle&);
292  // Handle the subscripts or mask.
293  // It checks if subscripts or mask was not already used.
294  void handleIndices (const TableExprNodeSet& indices,
295  const TaQLStyle& style);
296  ~TableParseUpdate();
297 
298  // Set the column name.
299  void setColumnName (const String& name);
300 
301  // Set the column name forthe mask.
302  void setColumnNameMask (const String& name);
303 
304  // Get the column name.
305  const String& columnName() const;
306 
307  // Get the possible column name for the mask.
308  const String& columnNameMask() const;
309 
310  // Tell if the mask is given first (i.e., before slice).
311  Bool maskFirst() const
312  { return maskFirst_p; }
313 
314  // Get the pointer to the indices.
315  TableExprNodeIndex* indexPtr() const;
316 
317  // Get the index expression node.
318  const TableExprNode& indexNode() const;
319 
320  // Get the expression node.
321  // <group>
322  const TableExprNode& node() const;
323  TableExprNode& node();
324  // </group>
325 
326  // Get the mask.
327  const TableExprNode& mask() const
328  { return mask_p; }
329 
330  // Adapt the possible unit of the expression to the possible unit
331  // of the column.
332  void adaptUnit (const Unit& columnUnit);
333 
334 private:
337  Bool maskFirst_p; //# True = mask is given before slice
338  TableExprNodeIndex* indexPtr_p; //# copy of pointer in indexNode_p
342 };
343 
344 
345 
346 
347 // <summary>
348 // Select-class for flex/bison scanner/parser for TableParse
349 // </summary>
350 
351 // <use visibility=local>
352 
353 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
354 // </reviewed>
355 
356 // <prerequisite>
357 //# Classes you should understand before using this one.
358 // <li> TableParse
359 // <li> TableGram.l and .y (flex and bison grammar)
360 // </prerequisite>
361 
362 // <synopsis>
363 // This class is needed for the the actions in the flex scanner
364 // and bison parser.
365 // This stores the information by constructing TableParse objects
366 // as needed and storing them in a vector.
367 // </synopsis>
368 
369 // <motivation>
370 // It is necessary to be able to give a table select command in ASCII.
371 // This can be used in a CLI or in the table browser to get a subset
372 // of a table or to sort a table.
373 // </motivation>
374 
375 //# <todo asof="$DATE:$">
376 //# A List of bugs, limitations, extensions or planned refinements.
377 //# </todo>
378 
379 
381 {
382 public:
383  enum CommandType {
391  PALTTAB
392  };
393 
395  GROUPBY=1,
396  AGGR_FUNCS=2,
397  ONLY_COUNTALL=4
398  };
399 
400  // Construct.
402 
403  // Destructor.
404  ~TableParseSelect();
405 
406  // Return the command type.
408  { return commandType_p; }
409 
410  // Return the expression node.
412  { return node_p; }
413 
414  // Create a temporary table in no tables are given in FROM.
415  void makeTableNoFrom (const vector<TableParseSelect*>& stack);
416 
417  // Execute the select command (select/sort/projection/groupby/having/giving).
418  // The setInGiving flag tells if a set in the GIVING part is allowed.
419  // The mustSelect flag tells if a SELECT command must do something.
420  // Usually that is required, but not for a SELECT in an INSERT command.
421  // Optionally the maximum nr of rows to be selected can be given.
422  // It will be used as the default value for the LIMIT clause.
423  // 0 = no maximum.
424  void execute (Bool showTimings, Bool setInGiving,
425  Bool mustSelect, uInt maxRow, Bool doTracing=False);
426 
427  // Execute a query in a from clause resulting in a Table.
428  Table doFromQuery (Bool showTimings);
429 
430  // Execute a subquery and create an appropriate node for the result.
431  TableExprNode doSubQuery (Bool showTimings);
432 
433  // Test if a subquery has sufficient elements.
434  // It uses default LIMIT=1, but that can be overidden in the subquery.
435  // The flag tells if NOT EXISTS or EXISTS was given.
436  TableExprNode doExists (Bool noexists, Bool showTimings);
437 
438  // Show the expression tree.
439  void show (ostream& os) const;
440 
441  // Keep the selection expression.
442  void handleWhere (const TableExprNode&);
443 
444  // Keep the groupby expressions.
445  // It checks if they are all scalar expressions.
446  void handleGroupby (const vector<TableExprNode>&, Bool rollup);
447 
448  // Keep the having expression.
449  void handleHaving (const TableExprNode&);
450 
451  // Keep the expression of a calculate command.
452  void handleCalcComm (const TableExprNode&);
453 
454  // Keep the create table command.
455  void handleCreTab (const Record& dmInfo);
456 
457  // Keep the column specification in a create table command.
458  void handleColSpec (const String& columnName, const String& dataType,
459  const Record& spec, Bool isCOrder=False);
460 
461  // Reopen the table (for update) used in the ALTER TABLE command.
462  void handleAltTab();
463 
464  // Add columns to the table of ALTER TABLE.
465  // The column descriptions have already been added to tableDesc_p.
466  void handleAddCol (const Record& dmInfo);
467 
468  // Add a keyword or replace a keyword with the value of another keyword.
469  // The keywords can be table or column keywords (col::key).
470  ValueHolder getRecFld (const String& name);
471 
472  // Define a field with the given data type in the Record.
473  static void setRecFld (RecordInterface& rec,
474  const String& name,
475  const String& dtype,
476  const ValueHolder& vh);
477 
478  // Get the type string. If empty, it is made from the given
479  // data type.
480  static String getTypeString (const String& typeStr, DataType type);
481 
482  // Add a keyword or replace a keyword with a value.
483  // The keyword can be a table or column keyword (col::key).
484  // The data type string can be empty leaving the data type unchanged.
485  void handleSetKey (const String& name, const String& dtype,
486  const ValueHolder& value);
487 
488  // Rename a table or column keyword.
489  void handleRenameKey (const String& oldName, const String& newName);
490 
491  // Remove a table or column keyword.
492  void handleRemoveKey (const String& name);
493 
494  // Split the given name into optional shorthand, column and fields.
495  // Find the keywordset for it and fill in the final keyword name.
496  // It is a helper function for handleSetKey, etc.
497  TableRecord& findKeyword (const String& name, String& keyName);
498 
499  // Add an update object.
500  void addUpdate (TableParseUpdate* upd);
501 
502  // Set the insert expressions for all rows.
503  void setInsertExprs (const std::vector<TableExprNode> exprs)
504  { insertExprs_p = exprs; }
505 
506  // Keep the update expressions.
507  void handleUpdate();
508 
509  // Make ready for the insert expression.
510  // The first one uses values (added via addUpdate),
511  // the second one a subquery.
512  // <group>
513  void handleInsert();
514  void handleInsert (TableParseSelect* sel);
515  // </group>
516 
517  // Make ready for a COUNT command.
518  // It checks if all column expressions are scalar.
519  void handleCount();
520 
521  // Keep the sort expressions.
522  void handleSort (const std::vector<TableParseSort>& sortList,
523  Bool noDuplicates, Sort::Order defaultSortOrder);
524 
525  // Evaluate and keep limit/offset/stride given as start:end:incr
526  void handleLimit (const TableExprNodeSetElem& expr);
527 
528  // Evaluate and keep the limit value.
529  void handleLimit (const TableExprNode& expr);
530 
531  // Evaluate and keep the offset value.
532  void handleOffset (const TableExprNode& expr);
533 
534  // Evaluate and add the rows.
535  void handleAddRow (const TableExprNode& expr);
536 
537  // Add a table nr, name, or object to the container.
538  void addTable (Int tabnr, const String& name,
539  const Table& table,
540  const String& shorthand,
541  const vector<const Table*> tempTables,
542  const vector<TableParseSelect*>& stack);
543 
544  // Make a Table object for given name, seqnr or so.
545  // If <src>alwaysOpen=False</src> the table will only be looked up,
546  // but not opened if not found. This is meant for concatenated tables
547  // in TaQLNodeHandler.
548  Table makeTable (Int tabnr, const String& name,
549  const Table& ftab,
550  const String& shorthand,
551  const vector<const Table*> tempTables,
552  const vector<TableParseSelect*>& stack,
553  Bool alwaysOpen=True);
554 
555  // Replace the first table (used by CALC command).
556  void replaceTable (const Table& table);
557 
558  // Find the keyword or column name and create a TableExprNode from it.
559  // If <src>tryProj=True</src> it is first tried if the column is a coluymn
560  // in the projected table (i.e., result from the SELECT part).
561  TableExprNode handleKeyCol (const String& name, Bool tryProj);
562 
563  // Handle a slice operator.
564  static TableExprNode handleSlice (const TableExprNode& array,
565  const TableExprNodeSet& indices,
566  const TaQLStyle&);
567 
568  // Handle a function.
569  TableExprNode handleFunc (const String& name,
570  const TableExprNodeSet& arguments,
571  const TaQLStyle&);
572 
573  // Make a function object node for the given function name and arguments.
574  // The ignoreFuncs vector contains invalid function codes.
575  static TableExprNode makeFuncNode (TableParseSelect*,
576  const String& name,
577  const TableExprNodeSet& arguments,
578  const Vector<int>& ignoreFuncs,
579  const Table& table,
580  const TaQLStyle&);
581 
582  // Add a column to the list of column names.
583  void handleColumn (Int type, const String& name, const TableExprNode& expr,
584  const String& newName, const String& nameMask,
585  const String& newDtype);
586 
587  // Finish the addition of columns to the list of column names.
588  void handleColumnFinish (Bool distinct);
589 
590  // Set the DataManager info for a new table.
591  void setDMInfo (const Record& dminfo)
592  { dminfo_p = dminfo;}
593 
594  // Handle the name and type given in a GIVING clause.
595  void handleGiving (const String& name, const Record& type);
596 
597  // Handle the set given in a GIVING clause.
598  void handleGiving (const TableExprNodeSet&);
599 
600  // Get the projected column names.
601  const Block<String>& getColumnNames() const;
602 
603  // Get the resulting table.
604  const Table& getTable() const;
605 
606  // An exception is thrown if the node uses an aggregate function.
607  static void checkAggrFuncs (const TableExprNode& node);
608 
609  // Split a name into its parts (shorthand, column and field names).
610  // True is returned if the name contained a keyword part.
611  // In that case fieldNames contains the keyword name and the possible
612  // subfields. The possible shorthand and the column name are
613  // filled in if it is a column keyword.
614  // If the name represents a column, fieldNames contains the subfields
615  // of the column (for the case where the column contains records).
616  // If the name is invalid, an exception is thrown if checkError=True.
617  // Otherwise the name is treated as a normal name without keyword.
618  // If allowEmtpy is True, :: is allowed, otherwise an error is thrown.
619  static Bool splitName (String& shorthand, String& columnName,
620  Vector<String>& fieldNames, const String& name,
621  Bool checkError, Bool isKeyword, Bool allowNoKey);
622 
623 private:
624  // Test if groupby or aggregate functions are given.
625  // <br> bit 0: on = groupby is given
626  // <br> bit 1: on = aggregate functions are given
627  // <br> bit 2: on = only select count(*) aggregate function is given
628  Int testGroupAggr (vector<TableExprNodeRep*>& aggr) const;
629 
630  // Get the aggregate functions used in SELECT and HAVING.
631  vector<TableExprNodeRep*> getAggrNodes() const;
632 
633  // Try to make a UDF function node for the given function name and arguments.
634  static TableExprNode makeUDFNode (TableParseSelect*,
635  const String& name,
636  const TableExprNodeSet& arguments,
637  const Table& table,
638  const TaQLStyle&);
639 
640  // Find the function code belonging to a function name.
641  // Functions to be ignored can be given (as function type values).
642  // If the function name is unknown, NRFUNC is returned.
643  static TableExprFuncNode::FunctionType findFunc
644  (const String& name,
645  uInt narguments,
646  const Vector<Int>& ignoreFuncs);
647 
648  // Do the update step.
649  // Rows 0,1,2,.. in UpdTable are updated from the expression result
650  // for the rows in the given rownrs vector.
651  void doUpdate (Bool showTimings, const Table& origTable,
652  Table& updTable, const Vector<uInt>& rownrs,
653  const CountedPtr<TableExprGroupResult>& groups =
655 
656  // Do the insert step and return a selection containing the new rows.
657  Table doInsert (Bool showTimings, Table& table);
658 
659  // Do the delete step.
660  void doDelete (Bool showTimings, Table& table);
661 
662  // Do the count step returning a memory table containing the unique
663  // column values and the counts of the column values.
664  Table doCount (Bool showTimings, const Table&);
665 
666  // Do the projection step returning a table containing the projection.
667  Table doProject (Bool showTimings, const Table&,
668  const CountedPtr<TableExprGroupResult>& groups =
670 
671  // Do the projection containing column expressions.
672  // Use the selected or unselected columns depending on <src>useSel</src>.
673  Table doProjectExpr (Bool useSel,
674  const CountedPtr<TableExprGroupResult>& groups);
675 
676  // Create a table using the given parameters.
677  // The variables set by handleGiven are used for name and type.
678  Table createTable (const TableDesc& td,
679  Int64 nrow, const Record& dmInfo);
680 
681  // Make the (empty) table for the epxression in the SELECT clause.
682  void makeProjectExprTable();
683 
684  // Fill projectExprSelColumn_p telling the columns to be projected
685  // at the first stage.
686  void makeProjectExprSel();
687 
688  // Add a column node to applySelNodes_p.
689  void addApplySelNode (const TableExprNode& node)
690  { applySelNodes_p.push_back (node); }
691 
692  // Set the selected rows for the column objects in applySelNodes_p.
693  // These nodes refer the original table. They requires different row
694  // numbers than the selected groups and projected columns.
695  // rownrs_p is changed to use row 0..n.
696  // It returns the Table containing the subset of rows in the input Table.
697  Table adjustApplySelNodes (const Table&);
698 
699  // Do the groupby/aggregate step and return its result.
701  (bool showTimings, vector<TableExprNodeRep*> aggrNodes,
702  Int groupAggrUsed);
703 
704  // Do the HAVING step.
705  void doHaving (Bool showTimings,
706  const CountedPtr<TableExprGroupResult>& groups);
707 
708  // Do a groupby/aggregate step that only does a 'select count(*)'.
709  CountedPtr<TableExprGroupResult> doOnlyCountAll (TableExprNodeRep* aggrNode);
710 
711  // Do a full groupby/aggregate step.
713  (const vector<TableExprNodeRep*>& aggrNodes);
714 
715  // Do the sort step.
716  void doSort (Bool showTimings);
717 
718  // Do the limit/offset step.
719  void doLimOff (Bool showTimings);
720  Table doLimOff (Bool showTimings, const Table& table);
721 
722  // Do the 'select distinct' step.
723  Table doDistinct (Bool showTimings, const Table& table);
724 
725  // Finish the table (rename, copy, and/or flush).
726  Table doFinish (Bool showTimings, Table& table);
727 
728  // Update the values in the columns (helpers of doUpdate).
729  // <group>
730  template<typename TCOL, typename TNODE>
731  void updateValue (uInt row, const TableExprId& rowid,
732  Bool isScalarCol, const TableExprNode& node,
733  const Array<Bool>& mask, Bool maskFirst,
734  TableColumn& col, const Slicer* slicerPtr,
735  ArrayColumn<Bool>& maskCol);
736  template<typename TCOL, typename TNODE>
737  void updateScalar (uInt row, const TableExprId& rowid,
738  const TableExprNode& node,
739  TableColumn& col);
740  template<typename TCOL, typename TNODE>
741  void updateArray (uInt row, const TableExprId& rowid,
742  const TableExprNode& node,
743  const Array<TNODE>& res,
744  ArrayColumn<TCOL>& col);
745  template<typename TCOL, typename TNODE>
746  void updateSlice (uInt row, const TableExprId& rowid,
747  const TableExprNode& node,
748  const Array<TNODE>& res,
749  const Slicer& slice,
750  ArrayColumn<TCOL>& col);
751  template<typename TCOL, typename TNODE>
752  void copyMaskedValue (uInt row, ArrayColumn<TCOL>& acol,
753  const Slicer* slicerPtr,
754  const TNODE* val,
755  uInt incr, const Array<Bool>& mask);
756  Array<Bool> makeMaskSlice (const Array<Bool>& mask,
757  Bool maskFirst,
758  const IPosition& shapeCol,
759  const Slicer* slicerPtr);
760  void checkMaskColumn (Bool hasMask,
761  const ArrayColumn<Bool>& maskCol,
762  const TableColumn& col);
763  // </group>
764 
765  // Make a data type from the string.
766  // It checks if it is compatible with the given (expression) data type.
767  DataType makeDataType (DataType dtype, const String& dtstr,
768  const String& colName);
769 
770  // Get the order for this key. Use the default order_p if not
771  // explicitly given with the key.
772  Sort::Order getOrder (const TableParseSort& key) const;
773 
774  // Make an array from the contents of a column in a subquery.
775  TableExprNode getColSet();
776 
777  // Make a set from the results of the subquery.
778  TableExprNode makeSubSet() const;
779 
780  // Evaluate an int scalar expression.
781  Int64 evalIntScaExpr (const TableExprNode& expr) const;
782 
783  // Find a table for the given shorthand.
784  // If no shorthand is given, the first table is returned (if there).
785  // If not found, a null Table object is returned.
786  Table findTable (const String& shorthand) const;
787 
788  // Handle the selection of a wildcarded column name.
789  void handleWildColumn (Int stringType, const String& name);
790 
791  // Add the description of a column to the table description.
792  // ndim < 0 means a scalar column.
793  void addColumnDesc (TableDesc& td, DataType dtype,
794  const String& colName, Int options,
795  Int ndim, const IPosition& shape,
796  const String& dmType, const String& dmGroup,
797  const String& comment,
798  const TableRecord& keywordSet,
799  const String& unitName);
800 
801  // Find the names of all stored columns in a table.
802  Block<String> getStoredColumns (const Table& tab) const;
803 
804  // Try to find the keyword representing a table in one of the tables
805  // in any select block (from inner to outer).
806  // If not found, an exception is thrown.
807  static Table tableKey (const String& fullName,
808  const String& shorthand, const String& columnName,
809  const Vector<String>& fieldNames,
810  const vector<TableParseSelect*>& stack);
811 
812  // Try to find the keyword representing a table in the given table.
813  // If the columnName is empty, the keyword is a table keyword.
814  // If not found, a null Table object is returned.
815  static Table findTableKey (const Table& table, const String& columnName,
816  const Vector<String>& keyNames);
817 
818  // Check if the tables used in selection columns have the same
819  // size as the first table given in FROM.
820  void checkTableProjSizes() const;
821 
822  // Create the set of aggregate functions and groupby keys in case
823  // a single groupby key is given.
824  // This offers much faster map access then doGroupByAggrMultiple.
825  template<typename T>
826  vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrSingleKey
827  (const vector<TableExprNodeRep*>& aggrNodes)
828  {
829  // We have to group the data according to the (possibly empty) groupby.
830  // We step through the table in the normal order which may not be the
831  // groupby order.
832  // A map<key,int> is used to keep track of the results where the int
833  // is the index in a vector of a set of aggregate function objects.
834  vector<CountedPtr<TableExprGroupFuncSet> > funcSets;
835  std::map<T, int> keyFuncMap;
836  T lastKey = std::numeric_limits<T>::max();
837  int groupnr = -1;
838  // Loop through all rows.
839  // For each row generate the key to get the right entry.
840  TableExprId rowid(0);
841  T key;
842  for (uInt i=0; i<rownrs_p.size(); ++i) {
843  rowid.setRownr (rownrs_p[i]);
844  groupbyNodes_p[0].get (rowid, key);
845  if (key != lastKey) {
846  typename std::map<T, int>::iterator iter = keyFuncMap.find (key);
847  if (iter == keyFuncMap.end()) {
848  groupnr = funcSets.size();
849  keyFuncMap[key] = groupnr;
850  funcSets.push_back (new TableExprGroupFuncSet (aggrNodes));
851  } else {
852  groupnr = iter->second;
853  }
854  }
855  rowid.setRownr (rownrs_p[i]);
856  funcSets[groupnr]->apply (rowid);
857  }
858  return funcSets;
859  }
860 
861  // Create the set of aggregate functions and groupby keys in case
862  // multiple keys are given.
863  vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrMultipleKeys
864  (const vector<TableExprNodeRep*>& aggrNodes);
865 
866  //# Command type.
868  //# Table description for a series of column descriptions.
870  //# Vector of TableParse objects.
871  //# This is needed for the functions above, otherwise they have no
872  //# way to communicate.
873  vector<TableParse> fromTables_p;
874  //# Block of selected column names (new name in case of select).
876  //# Block of selected mask column names (for masked arrays).
878  //# Block of selected column expressions.
880  //# The old name for a selected column.
882  //# The new data type for a column.
884  //# The keywords used in a column.
886  //# Number of real expressions used in selected columns.
888  //# Distinct values in output?
890  //# Name and type of the resulting table (from GIVING part).
892  uInt resultType_p; //# 0-unknown 1=memory 2=scratch 3=plain
893  Bool resultCreated_p; //# Has the result table been created?
898  //# Resulting set (from GIVING part).
900  //# The WHERE expression tree.
902  //# The GROUPBY expressions.
903  vector<TableExprNode> groupbyNodes_p;
904  Bool groupbyRollup_p; //# use ROLLUP in GROUPBY?
905  //# The HAVING expression.
907  //# The possible limit (= max nr of selected rows) (0 means no limit).
909  //# The possible last row (0 means no end; can be <0).
910  //# limit_p and endrow_p cannot be both !=0.
912  //# The possible offset (= nr of selected rows to skip).
914  //# The possible stride in offset:endrow:stride.
916  //# The update and insert list.
917  std::vector<TableParseUpdate*> update_p;
918  //# The insert expressions (possibly for multiple rows).
919  std::vector<TableExprNode> insertExprs_p;
920  //# The table selection to be inserted.
922  //# The sort list.
923  std::vector<TableParseSort> sort_p;
924  //# The noDuplicates sort switch.
926  //# The default sort order.
928  //# All nodes that need to be adjusted for a selection of rownrs.
929  //# It can consist of column nodes and the rowid function node.
930  //# Some nodes (in aggregate functions) can later be disabled for adjustment.
931  vector<TableExprNode> applySelNodes_p;
932  //# The resulting table.
934  //# The first table used when creating a column object.
935  //# All other tables used for them should have the same size.
938  //# The table resulting from a projection with expressions.
940  //# The projected columns used in the HAVING and ORDERBY clauses.
943  //# The resulting row numbers.
945 };
946 
947 
948 
949 //# Implement the inline functions.
950 inline Bool TableParse::test (const String& str) const
951  { return (shorthand_p == str ? True : False); }
952 
953 inline const String& TableParse::shorthand() const
954  { return shorthand_p; }
955 
956 inline const Table& TableParse::table() const
957  { return table_p; }
958 
959 
960 inline void TableParseUpdate::setColumnName (const String& name)
961  { columnName_p = name; }
963  { columnNameMask_p = name; }
965  { return columnName_p; }
967  { return columnNameMask_p; }
969  { return indexPtr_p; }
971  { return indexNode_p; }
973  { return node_p; }
975  { return node_p; }
976 inline void TableParseUpdate::adaptUnit (const Unit& columnUnit)
977  { node_p.adaptUnit (columnUnit); }
978 
979 inline const TableExprNode& TableParseSort::node() const
980  { return node_p; }
982  { return given_p; }
984  { return order_p; }
985 
986 
988  { return columnNames_p; }
989 
990 inline const Table& TableParseSelect::getTable() const
991  { return table_p; }
992 
994  { update_p.push_back (upd); }
995 
997  { return (key.orderGiven() ? key.order() : order_p); }
998 
999 
1000 } //# NAMESPACE CASACORE - END
1001 
1002 #endif
A Vector of integers, for indexing into Array<T> objects.
Definition: IPosition.h:119
Block< Bool > projectExprSelColumn_p
Definition: TableParse.h:942
A 1-D Specialization of the Array class.
Definition: ArrayIO.h:45
vector< TableExprNode > groupbyNodes_p
Definition: TableParse.h:903
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
void setColumnName(const String &name)
Set the column name.
Definition: TableParse.h:960
int Int
Definition: aipstype.h:50
TableExprNodeSet * resultSet_p
Definition: TableParse.h:899
Bool orderGiven() const
Is the order given?
Definition: TableParse.h:981
EndianFormat
Define the possible endian formats in which table data can be stored.
Definition: Table.h:188
Main interface class to a read/write table.
Definition: Table.h:149
TableExprNodeIndex * indexPtr_p
Definition: TableParse.h:338
LatticeExprNode mask(const LatticeExprNode &expr)
This function returns the mask of the given expression.
std::vector< TableExprNode > insertExprs_p
Definition: TableParse.h:919
TableExprNode indexNode_p
Definition: TableParse.h:339
TableExprNode array(const TableExprNode &values, const TableExprNodeSet &shape)
Create an array of the given shape and fill it with the values.
Definition: ExprNode.h:2093
Class to hold multiple table expression nodes.
Definition: ExprNodeSet.h:306
LatticeExprNode max(const LatticeExprNode &left, const LatticeExprNode &right)
Handle class for a table column expression tree.
Definition: ExprNode.h:614
TableExprNode node_p
Definition: TableParse.h:228
Order
Enumerate the sort order:
Definition: Sort.h:260
void setRownr(uInt rownr)
Set the row number.
Definition: TableExprId.h:186
void setColumnNameMask(const String &name)
Set the column name forthe mask.
Definition: TableParse.h:962
TableExprNodeIndex * indexPtr() const
Get the pointer to the indices.
Definition: TableParse.h:968
Sort::Order getOrder(const TableParseSort &key) const
Get the order for this key.
Definition: TableParse.h:996
void adaptUnit(const Unit &columnUnit)
Adapt the possible unit of the expression to the possible unit of the column.
Definition: TableParse.h:976
Abstract base class for a node in a table column expression tree.
Definition: ExprNodeRep.h:151
TableParse & operator=(const TableParse &)
Assignment (copy semantics).
Block< String > columnNameMasks_p
Definition: TableParse.h:877
const Table & table() const
Get table object.
Definition: TableParse.h:956
Options defining how table files are organized.
Definition: StorageOption.h:71
defines physical units
Definition: Unit.h:189
vector< TableExprNode > applySelNodes_p
Definition: TableParse.h:931
vector< TableParse > fromTables_p
Definition: TableParse.h:873
Select-class for flex/bison scanner/parser for TableParse.
Definition: TableParse.h:380
Sort::Order order() const
Get the sort order.
Definition: TableParse.h:983
Referenced counted pointer for constant data.
Definition: CountedPtr.h:86
Bool maskFirst() const
Tell if the mask is given first (i.e., before slice).
Definition: TableParse.h:311
const TableExprNode & node() const
Get the expression node.
Definition: TableParse.h:972
Table::EndianFormat endianFormat_p
Definition: TableParse.h:895
Class to hold the table expression nodes for an element in a set.
Definition: ExprNodeSet.h:94
Class to hold the result of a TaQL command.
Definition: TaQLResult.h:67
Vector< uInt > rownrs_p
Definition: TableParse.h:944
A holder for a value of any basic Casacore data type.
Definition: ValueHolder.h:67
Class with static members defining the TaQL style.
Definition: TaQLStyle.h:64
LatticeExprNode ndim(const LatticeExprNode &expr)
1-argument function to get the dimensionality of a lattice.
CommandType commandType() const
Return the command type.
Definition: TableParse.h:407
Block< uInt > projectExprSubset_p
Definition: TableParse.h:941
Block< String > columnDtypes_p
Definition: TableParse.h:883
const TableExprNode & mask() const
Get the mask.
Definition: TableParse.h:327
const Block< String > & getColumnNames() const
Get the projected column names.
Definition: TableParse.h:987
const String & columnNameMask() const
Get the possible column name for the mask.
Definition: TableParse.h:966
Block< TableRecord > columnKeywords_p
Definition: TableParse.h:885
A hierarchical collection of named fields of various types.
Definition: Record.h:180
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
Bool test(const String &shortHand) const
Test if shorthand matches.
Definition: TableParse.h:950
Read/write access to a table column.
Definition: TableColumn.h:98
Class containing the results of aggregated values in a group.
Definition: ExprGroup.h:801
Block< String > columnNames_p
Definition: TableParse.h:875
const Bool False
Definition: aipstype.h:44
TableExprNode shape(const TableExprNode &array)
Function operating on any scalar or array resulting in a Double array containing the shape...
Definition: ExprNode.h:2151
A hierarchical collection of named fields of various types.
Definition: TableRecord.h:182
Block< TableExprNode > columnExpr_p
Definition: TableParse.h:879
TableExprNode getNode() const
Return the expression node.
Definition: TableParse.h:411
TableParse()
Default constructor for container class.
Helper class for sort keys in TableParse.
Definition: TableParse.h:198
Specify which elements to extract from an n-dimensional array.
Definition: Slicer.h:289
simple 1-D array
Definition: ArrayIO.h:47
const TableExprNode & node() const
Get the expression node.
Definition: TableParse.h:979
Class to hold values from table grammar parser.
Definition: TableParse.h:108
TableExprNode havingNode_p
Definition: TableParse.h:906
TableParseSelect * insSel_p
Definition: TableParse.h:921
The identification of a TaQL selection subject.
Definition: TableExprId.h:98
void addUpdate(TableParseUpdate *upd)
Add an update object.
Definition: TableParse.h:993
void setInsertExprs(const std::vector< TableExprNode > exprs)
Set the insert expressions for all rows.
Definition: TableParse.h:503
Block< String > columnOldNames_p
Definition: TableParse.h:881
const TableExprNode & indexNode() const
Get the index expression node.
Definition: TableParse.h:970
String: the storage and methods of handling collections of characters.
Definition: String.h:223
Define the structure of a Casacore table.
Definition: TableDesc.h:186
const Table & getTable() const
Get the resulting table.
Definition: TableParse.h:990
TaQLResult tableCommand(const String &command)
Abstract base class for Record classes.
StorageOption storageOption_p
Definition: TableParse.h:894
const String & shorthand() const
Get the shorthand.
Definition: TableParse.h:953
The index of an array element in a table select expression.
void addApplySelNode(const TableExprNode &node)
Add a column node to applySelNodes_p.
Definition: TableParse.h:689
const Bool True
Definition: aipstype.h:43
const String & columnName() const
Get the column name.
Definition: TableParse.h:964
this file contains all the compiler specific defines
Definition: mainpage.dox:28
LatticeExprNode value(const LatticeExprNode &expr)
This function returns the value of the expression without a mask.
unsigned int uInt
Definition: aipstype.h:51
Helper class for updates in TableParse.
Definition: TableParse.h:262
std::vector< TableParseUpdate * > update_p
Definition: TableParse.h:917
std::vector< TableParseSort > sort_p
Definition: TableParse.h:923
void setDMInfo(const Record &dminfo)
Set the DataManager info for a new table.
Definition: TableParse.h:591