EMMA Coverage Report (generated Wed Jun 28 22:15:27 PDT 2006)
[all classes][org.apache.derby.impl.store.raw.log]

COVERAGE SUMMARY FOR SOURCE FILE [LogToFile.java]

nameclass, %method, %block, %line, %
LogToFile.java100% (1/1)90%  (81/90)76%  (3868/5106)76%  (958.2/1263)

COVERAGE BREAKDOWN BY CLASS AND METHOD

nameclass, %method, %block, %line, %
     
class LogToFile100% (1/1)90%  (81/90)76%  (3868/5106)76%  (958.2/1263)
<static initializer> 100% (1/1)100% (27/27)100% (13/13)
LogToFile (): void 100% (1/1)100% (75/75)100% (25/25)
appendLogRecord (byte [], int, int, byte [], int, int): long 100% (1/1)56%  (105/187)71%  (22.7/32)
boot (boolean, Properties): void 100% (1/1)73%  (278/382)70%  (65.8/94)
canSupport (Properties): boolean 100% (1/1)57%  (8/14)60%  (3/5)
checkCorrupt (): void 100% (1/1)52%  (11/21)67%  (3.4/5)
checkVersion (int, int): boolean 100% (1/1)78%  (14/18)60%  (3/5)
checkpoint (RawStoreFactory, DataFactory, TransactionFactory, boolean): boolean 100% (1/1)35%  (13/37)27%  (3/11)
checkpointInRFR (LogInstant, long, DataFactory): void 100% (1/1)60%  (18/30)62%  (5/8)
checkpointWithTran (RawTransaction, RawStoreFactory, DataFactory, Transaction... 100% (1/1)74%  (180/242)80%  (52.2/65)
copyActiveLogFiles (File): boolean 100% (1/1)91%  (83/91)80%  (16/20)
createLogDirectory (): void 100% (1/1)85%  (29/34)89%  (8/9)
currentInstant (): long 100% (1/1)100% (6/6)100% (1/1)
databaseEncrypted (): boolean 100% (1/1)100% (3/3)100% (1/1)
decrypt (byte [], int, int, byte [], int): int 100% (1/1)100% (9/9)100% (1/1)
deleteObsoleteLogfiles (): void 100% (1/1)68%  (74/109)65%  (15/23)
deleteOnlineArchivedLogFiles (): void 100% (1/1)100% (3/3)100% (2/2)
disableLogArchiveMode (): void 100% (1/1)100% (23/23)100% (7/7)
enableLogArchiveMode (): void 100% (1/1)100% (26/26)100% (8/8)
encrypt (byte [], int, int, byte [], int): int 100% (1/1)100% (9/9)100% (1/1)
endPosition (): long 0%   (0/1)0%   (0/3)0%   (0/1)
findCheckpoint (long, FileLogger): CheckpointOperation 100% (1/1)91%  (20/22)83%  (5/6)
firstLogInstant (): long 0%   (0/1)0%   (0/5)0%   (0/1)
flush (LogInstant): void 100% (1/1)100% (21/21)100% (8/8)
flush (long, long): void 100% (1/1)74%  (246/334)77%  (55.9/73)
flushAll (): void 100% (1/1)78%  (18/23)95%  (5.7/6)
flushBuffer (long, long): void 100% (1/1)84%  (27/32)94%  (7.5/8)
freezePersistentStore (): void 100% (1/1)76%  (16/21)92%  (5.5/6)
getCanonicalLogPath (): String 100% (1/1)75%  (9/12)60%  (3/5)
getControlFileName (): StorageFile 100% (1/1)100% (7/7)100% (1/1)
getEncryptedDataLength (int): int 100% (1/1)88%  (15/17)67%  (2/3)
getEncryptionBlockSize (): int 100% (1/1)100% (4/4)100% (1/1)
getFirstLogNeeded (CheckpointOperation): long 100% (1/1)91%  (50/55)93%  (8.4/9)
getFirstUnflushedInstant (): LogInstant 100% (1/1)95%  (21/22)98%  (2/2)
getLogDirPath (StorageFile): String 0%   (0/1)0%   (0/21)0%   (0/3)
getLogDirectory (): StorageFile 100% (1/1)72%  (13/18)80%  (4/5)
getLogFactoryProperties (PersistentSet): void 100% (1/1)77%  (46/60)78%  (14/18)
getLogFileAtBeginning (long): StorageRandomAccessFile 100% (1/1)100% (8/8)100% (2/2)
getLogFileAtPosition (long): StorageRandomAccessFile 100% (1/1)53%  (50/95)54%  (14/26)
getLogFileName (long): StorageFile 100% (1/1)100% (16/16)100% (1/1)
getLogFileNumber (): long 100% (1/1)100% (3/3)100% (1/1)
getLogFileToSimulateCorruption (long): StorageRandomAccessFile 100% (1/1)100% (11/11)100% (3/3)
getLogStorageFactory (): void 100% (1/1)79%  (30/38)70%  (7/10)
getLogger (): Logger 100% (1/1)80%  (8/10)67%  (2/3)
getMirrorControlFileName (): StorageFile 100% (1/1)100% (7/7)100% (1/1)
getTypeFormatId (): int 0%   (0/1)0%   (0/2)0%   (0/1)
inRFR (): boolean 100% (1/1)82%  (31/38)81%  (6.5/8)
initLogFile (StorageRandomAccessFile, long, long): boolean 100% (1/1)78%  (31/40)77%  (10/13)
logArchived (): boolean 100% (1/1)100% (3/3)100% (1/1)
logErrMsg (String): void 100% (1/1)100% (9/9)100% (5/5)
logErrMsg (Throwable): void 100% (1/1)72%  (18/25)70%  (7/10)
logErrMsgForDurabilityTestModeNoSync (): void 100% (1/1)60%  (6/10)67%  (2/3)
logtest_appendPartialLogRecord (byte [], int, int, byte [], int, int): long 100% (1/1)96%  (238/248)95%  (46.6/49)
markCorrupt (StandardException): StandardException 100% (1/1)84%  (59/70)90%  (17.9/20)
openBackwardsScan (LogInstant): LogScan 100% (1/1)82%  (23/28)96%  (5.8/6)
openBackwardsScan (long, LogInstant): LogScan 100% (1/1)83%  (20/24)80%  (4/5)
openFlushedScan (DatabaseInstant, int): ScanHandle 0%   (0/1)0%   (0/7)0%   (0/1)
openForwardsFlushedScan (LogInstant): LogScan 0%   (0/1)0%   (0/10)0%   (0/2)
openForwardsScan (LogInstant, LogInstant): LogScan 0%   (0/1)0%   (0/21)0%   (0/6)
openForwardsScan (long, LogInstant): LogScan 100% (1/1)55%  (26/47)58%  (6.4/11)
openLogFileInWriteMode (StorageFile): StorageRandomAccessFile 100% (1/1)47%  (8/17)50%  (3/6)
performWork (ContextManager): int 100% (1/1)75%  (49/65)87%  (16.6/19)
preAllocateNewLogFile (StorageRandomAccessFile): void 100% (1/1)93%  (54/58)87%  (14/16)
printErrorStack (Throwable): void 100% (1/1)100% (16/16)100% (5/5)
privCanWrite (StorageFile): boolean 100% (1/1)100% (5/5)100% (1/1)
privCopyFile (File, StorageFile): boolean 100% (1/1)82%  (14/17)67%  (4/6)
privCopyFile (StorageFile, File): boolean 100% (1/1)82%  (14/17)67%  (4/6)
privDelete (StorageFile): boolean 100% (1/1)100% (5/5)100% (1/1)
privExists (StorageFile): boolean 100% (1/1)100% (5/5)100% (1/1)
privList (File): String [] 100% (1/1)77%  (10/13)60%  (3/5)
privList (StorageFile): String [] 100% (1/1)77%  (10/13)60%  (3/5)
privMkdirs (StorageFile): boolean 100% (1/1)100% (5/5)100% (1/1)
privRandomAccessFile (StorageFile, String): StorageRandomAccessFile 100% (1/1)72%  (13/18)67%  (4/6)
privRemoveDirectory (StorageFile): boolean 100% (1/1)100% (5/5)100% (1/1)
readControlFile (StorageFile, Properties): long 100% (1/1)62%  (200/325)68%  (46.1/68)
recover (RawStoreFactory, DataFactory, TransactionFactory): void 100% (1/1)79%  (572/728)73%  (127.6/176)
restoreLogs (Properties): boolean 100% (1/1)88%  (115/130)91%  (31/34)
run (): Object 100% (1/1)97%  (67/69)93%  (13/14)
runBooleanAction (int, StorageFile): boolean 100% (1/1)79%  (11/14)60%  (3/5)
serviceASAP (): boolean 0%   (0/1)0%   (0/2)0%   (0/1)
serviceImmediately (): boolean 0%   (0/1)0%   (0/2)0%   (0/1)
stop (): void 100% (1/1)88%  (58/66)89%  (16.1/18)
switchLogFile (): void 100% (1/1)76%  (228/300)73%  (52.4/72)
syncFile (StorageRandomAccessFile): void 100% (1/1)32%  (7/22)33%  (4/12)
testLogFull (): void 100% (1/1)93%  (43/46)88%  (7/8)
truncateLog (CheckpointOperation): void 100% (1/1)51%  (41/81)63%  (12/19)
unfreezePersistentStore (): void 100% (1/1)72%  (13/18)94%  (4.7/5)
verifyLogFormat (StorageFile, long): boolean 100% (1/1)94%  (17/18)86%  (6/7)
verifyLogFormat (StorageRandomAccessFile, long): boolean 100% (1/1)65%  (22/34)70%  (7/10)
writeControlFile (StorageFile, long): boolean 100% (1/1)79%  (157/199)82%  (44.5/54)

1/*
2 
3   Derby - Class org.apache.derby.impl.store.raw.log.LogToFile
4 
5   Copyright 1997, 2004 The Apache Software Foundation or its licensors, as applicable.
6 
7   Licensed under the Apache License, Version 2.0 (the "License");
8   you may not use this file except in compliance with the License.
9   You may obtain a copy of the License at
10 
11      http://www.apache.org/licenses/LICENSE-2.0
12 
13   Unless required by applicable law or agreed to in writing, software
14   distributed under the License is distributed on an "AS IS" BASIS,
15   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   See the License for the specific language governing permissions and
17   limitations under the License.
18 
19 */
20 
21package org.apache.derby.impl.store.raw.log;
22 
23import org.apache.derby.iapi.services.diag.Performance;
24 
25import org.apache.derby.impl.store.raw.log.CheckpointOperation;
26import org.apache.derby.impl.store.raw.log.LogCounter;
27import org.apache.derby.impl.store.raw.log.LogRecord;
28import org.apache.derby.impl.store.raw.log.StreamLogScan;
29 
30// need this to print nested exception that corrupts the database
31import org.apache.derby.iapi.services.context.ErrorStringBuilder;
32 
33import org.apache.derby.iapi.services.info.ProductGenusNames;
34import org.apache.derby.iapi.services.info.ProductVersionHolder;
35 
36import org.apache.derby.iapi.reference.MessageId;
37import org.apache.derby.iapi.reference.Property;
38import org.apache.derby.iapi.reference.SQLState;
39 
40import org.apache.derby.iapi.services.daemon.DaemonService;
41import org.apache.derby.iapi.services.daemon.Serviceable;
42import org.apache.derby.iapi.services.context.ContextManager;
43import org.apache.derby.iapi.services.context.ContextService;
44import org.apache.derby.iapi.services.context.ShutdownException;
45import org.apache.derby.iapi.services.monitor.Monitor;
46import org.apache.derby.iapi.services.monitor.ModuleControl;
47import org.apache.derby.iapi.services.monitor.ModuleSupportable;
48import org.apache.derby.iapi.services.monitor.PersistentService;
49import org.apache.derby.iapi.services.sanity.SanityManager;
50import org.apache.derby.iapi.services.io.Formatable;
51import org.apache.derby.iapi.services.io.TypedFormat;
52import org.apache.derby.iapi.services.io.FormatIdUtil;
53import org.apache.derby.iapi.services.io.StoredFormatIds;
54import org.apache.derby.iapi.services.stream.HeaderPrintWriter;
55import org.apache.derby.iapi.services.stream.PrintWriterGetHeader;
56import org.apache.derby.iapi.services.stream.InfoStreams;
57import org.apache.derby.iapi.error.StandardException;
58import org.apache.derby.iapi.services.i18n.MessageService;
59import org.apache.derby.iapi.store.access.AccessFactory;
60import org.apache.derby.iapi.store.access.AccessFactoryGlobals;
61import org.apache.derby.iapi.store.access.TransactionController;
62import org.apache.derby.iapi.store.raw.Loggable;
63import org.apache.derby.iapi.store.raw.RawStoreFactory;
64import org.apache.derby.iapi.store.raw.ScanHandle;
65import org.apache.derby.iapi.store.raw.log.LogFactory;
66import org.apache.derby.iapi.store.raw.log.Logger;
67import org.apache.derby.iapi.store.raw.log.LogInstant;
68import org.apache.derby.iapi.store.raw.log.LogScan;
69import org.apache.derby.iapi.store.raw.Transaction;
70import org.apache.derby.iapi.store.raw.xact.RawTransaction;
71import org.apache.derby.iapi.store.raw.xact.TransactionFactory;
72import org.apache.derby.iapi.store.raw.data.DataFactory;
73import org.apache.derby.iapi.services.property.PersistentSet;
74 
75import org.apache.derby.iapi.store.access.DatabaseInstant;
76import org.apache.derby.catalog.UUID;
77import org.apache.derby.iapi.services.uuid.UUIDFactory;
78import org.apache.derby.iapi.services.property.PropertyUtil;
79import org.apache.derby.iapi.reference.Attribute;
80import org.apache.derby.iapi.services.io.FileUtil;
81import org.apache.derby.iapi.util.ReuseFactory;
82 
83import org.apache.derby.io.StorageFactory;
84import org.apache.derby.io.WritableStorageFactory;
85import org.apache.derby.io.StorageFile;
86import org.apache.derby.io.StorageRandomAccessFile;
87 
88import java.io.File; // Plain files are used for backups
89import java.io.IOException;
90import java.io.SyncFailedException;
91import java.io.ByteArrayOutputStream;
92import java.io.DataOutputStream;
93import java.io.ByteArrayInputStream;
94import java.io.DataInputStream;
95import java.io.FileNotFoundException;
96 
97import java.net.MalformedURLException;
98import java.net.URL;
99 
100import java.util.Properties;
101import java.util.Vector;
102import java.util.zip.CRC32;
103 
104/**
105 
106        This is an implementation of the log using a non-circular file system file.
107        No support for incremental log backup or media recovery.
108        Only crash recovery is supported.  
109        <P>
110        The 'log' is a stream of log records.  The 'log' is implemented as
111        a series of numbered log files.  These numbered log files are logically
112        continuous so a transaction can have log records that span multiple log files.
113        A single log record cannot span more then one log file.  The log file number
114        is monotonically increasing.
115        <P>
116        The log belongs to a log factory of a RawStore.  In the current implementation,
117        each RawStore only has one log factory, so each RawStore only has one log
118        (which composed of multiple log files).
119        At any given time, a log factory only writes new log records to one log file,
120        this log file is called the 'current log file'.
121        <P>
122        A log file is named log<em>logNumber</em>.dat
123        <P>
124        Everytime a checkpoint is taken, a new log file is created and all subsequent
125        log records will go to the new log file.  After a checkpoint is taken, old
126        and useless log files will be deleted.
127        <P>
128        RawStore exposes a checkpoint method which clients can call, or a checkpoint is
129        taken automatically by the RawStore when
130        <OL>
131        <LI> the log file grows beyond a certain size (configurable, default 100K bytes)
132        <LI> RawStore is shutdown and a checkpoint hasn't been done "for a while"
133        <LI> RawStore is recovered and a checkpoint hasn't been done "for a while"
134        </OL>
135        <P>
136        This LogFactory is responsible for the formats of 2 kinds of file: the log
137        file and the log control file.  And it is responsible for the format of the
138        log record wrapper.
139        <P> <PRE>
140 
141        Format of log control file 
142 
143        @format_id        FILE_STREAM_LOG_FILE
144        @purpose        The log control file contains information about which log files
145        are present and where the last checkpoint log record is located.
146        @upgrade        
147        @disk_layout
148        (pre-v15)
149                int format id
150                int log file version
151                long the log instant (LogCounter) of the last completed checkpoint
152        (v15 onward)
153                int format id
154                int obsolete log file version
155                long the log instant (LogCounter) of the last completed checkpoint
156                int JBMS version
157                int checkpoint interval
158                long spare (value set to 0)
159                long spare (value set to 0)
160                long spare (value set to 0)
161 
162        @end_format
163        </PRE>        
164        <HR WIDTH="100%">
165        <PRE>
166 
167        Format of the log file
168 
169        @format_id        FILE_STREAM_LOG_FILE
170        @purpose        The log file contains log record which record all the changes
171        to the database.  The complete transaction log is composed of a series of
172        log files.
173        @upgrade
174        @disk_layout
175                int format id -         the format Id of this log file
176                int obsolete log file version - not used
177                long log file number - this number orders the log files in a
178                                                series to form the complete transaction log
179                long prevLogRecord - log instant of the previous log record, in the
180                                previous log file. 
181 
182                [log record wrapper]* one or more log records with wrapper
183 
184                int endMarker - value of zero.  The beginning of a log record wrapper
185                                is the length of the log record, therefore it is never zero
186                [int fuzzy end]* zero or more int's of value 0, in case this log file
187                                has been recovered and any incomplete log record set to zero.
188        @end_format
189        </PRE>        
190        <HR WIDTH="100%">
191        <PRE>
192 
193        Format of the log record wrapper
194 
195        @format_id none.  The format is implied by the FILE_STREAM_LOG_FILE
196        @purpose        The log record wrapper provides information for the log scan.
197        @upgrade
198        @disk_layout
199                length(int) length of the log record (for forward scan)
200                instant(long) LogInstant of the log record
201                logRecord(byte[length]) byte array that is written by the FileLogger
202                length(int) length of the log record (for backward scan)
203        @end_format
204        </PRE>
205 
206 
207        <P>Multithreading considerations:<BR>
208        Log Factory must be MT-safe.
209        <P>
210        Class is final as it has methods with privilege blocks
211        and implements PrivilegedExceptionAction.
212        */
213 
214public final class LogToFile implements LogFactory, ModuleControl, ModuleSupportable,
215                                                                  Serviceable, java.security.PrivilegedExceptionAction
216{
217 
218        private static int fid = StoredFormatIds.FILE_STREAM_LOG_FILE; 
219 
220        // format Id must fit in 4 bytes
221 
222        /**
223                Return my format identifier.
224        */
225        public int getTypeFormatId() {
226                return StoredFormatIds.FILE_STREAM_LOG_FILE;
227        }
228 
229        // at the beginning of every log file is the following information:
230        // the log file formatId
231        // the log file version (int)
232        // the log file number (long)
233        // the log instant at the end of the last log record in the previous file (long)
234        public static final int LOG_FILE_HEADER_SIZE = 24;
235 
236        protected static final int LOG_FILE_HEADER_PREVIOUS_LOG_INSTANT_OFFSET = LOG_FILE_HEADER_SIZE-8;
237 
238        // Number of bytes overhead of each log record.
239        // 4 bytes of length at the beginning, 8 bytes of log instant,
240        // and 4 bytes of length at the end
241        public static final int LOG_RECORD_OVERHEAD = 4+8+4;
242 
243        public static final String DBG_FLAG = SanityManager.DEBUG ? "LogTrace" : null;
244        public static final String DUMP_LOG_ONLY = SanityManager.DEBUG ? "DumpLogOnly" : null;
245        public static final String DUMP_LOG_FROM_LOG_FILE = 
246                SanityManager.DEBUG ? "derby.storage.logDumpStart" : null;
247        protected static final String LOG_SYNC_STATISTICS = "LogSyncStatistics";
248 
249        // If you change this number, then JBMS 1.1x and 1.2x will give a really
250        // horrendous error message when booting against a db created by you.  When
251        // we decided that we don't need to worry about people mis-using the
252        // product that way, then we can change this.  Just remember, before we do,
253        // all existing database will have the number 9 in there.
254        private static final int OBSOLETE_LOG_VERSION_NUMBER = 9;
255 
256        /* how big the log file should be before checkpoint or log switch is taken */
257        private static final int DEFAULT_LOG_SWITCH_INTERVAL = 1024*1024;                
258        private static final int LOG_SWITCH_INTERVAL_MIN     = 100000;
259        private static final int LOG_SWITCH_INTERVAL_MAX     = 128*1024*1024;
260        private static final int CHECKPOINT_INTERVAL_MIN     = 100000;
261        private static final int CHECKPOINT_INTERVAL_MAX     = 128*1024*1024;
262        private static final int DEFAULT_CHECKPOINT_INTERVAL = 10*1024*1024;
263 
264        //log buffer size values
265        private static final int DEFAULT_LOG_BUFFER_SIZE = 32768; //32K
266        private static final int LOG_BUFFER_SIZE_MIN = 8192; //8k
267        private static final int LOG_BUFFER_SIZE_MAX = LOG_SWITCH_INTERVAL_MAX;
268        private int logBufferSize = DEFAULT_LOG_BUFFER_SIZE;
269 
270        /* Log Control file flags. */
271        private static final byte IS_BETA_FLAG = 0x1;
272        
273    /**
274     * When the derby.system.durability property is set to 'test', the store 
275     * system will not force sync calls in the following cases
276     * - for the log file at each commit
277     * - for the log file before data page is forced to disk
278     * - for page allocation when file is grown
279     * - for data writes during checkpoint
280     * This means it is possible that the recovery system may not work properly,
281     * committed transactions may be lost, and/or database may not
282     * be in a consistent state.
283     * In order that we recognize this case that the database was previously
284     * at any time booted in this mode, this value is written out
285     * into the log control file. This will help prevent us from 
286     * wasting time to resolve issues in such cases. 
287     * @see org.apache.derby.iapi.reference.Property#DURABILITY_PROPERTY
288     * This value is written as part of the log control file flags byte.
289     */
290    private static final byte IS_DURABILITY_TESTMODE_NO_SYNC_FLAG = 0x2;
291        
292    /**
293     * keeps track of if the database was booted previously at any time with 
294     * derby.system.durability=test
295     */
296    private static boolean wasDBInDurabilityTestModeNoSync = false;
297    
298        /* to err on the conservative side, unless otherwise set, assume log
299         *        archive is ON 
300         */
301        private static final String DEFAULT_LOG_ARCHIVE_DIRECTORY = "DEFAULT";
302 
303        private int     logSwitchInterval   = DEFAULT_LOG_SWITCH_INTERVAL;
304        private int     checkpointInterval  = DEFAULT_CHECKPOINT_INTERVAL;
305 
306        String dataDirectory;                                         // where files are stored
307    private WritableStorageFactory logStorageFactory;
308    
309        private boolean logBeingFlushed; // is the log in the middle of a flush
310                                                                         // (access of the variable should sync on this)
311 
312        protected LogAccessFile logOut;                // an output stream to the log file
313                                                                // (access of the variable should sync on this)
314        private   StorageRandomAccessFile firstLog = null;
315        protected long                     endPosition = -1; // end position of the current log file
316        long                                         lastFlush = 0;        // the position in the current log
317                                                                                        // file that has been flushed to disk
318 
319        long                                         logFileNumber = -1; // current log file number
320                                                                // other than during boot and recovery time,
321                                                                // logFileNumber is only changed by
322                                                                // switchLogFile, which is synchronized.
323                                                                // 
324                                                                // MT - Anyone accessing this number should
325                                                                // synchronized on this if the current log file
326                                                                // must not be changed. If not synchronized,
327                                                                // the log file may have been switched.
328 
329        long                                         firstLogFileNumber = -1;
330                                                                // first log file that makes up the active
331                                                                // portion (with active transactions) of the
332                                                                // log.  
333                                                                // 
334                                                                // MT - This value is set during recovery or
335                                                                // during log truncation.  In the former single
336                                                                // thread is assumed.  In the latter
337                                                                // must be synchronized with this to access
338                                                                // or change.
339        
340        private long              maxLogFileNumber = LogCounter.MAX_LOGFILE_NUMBER;
341        private CheckpointOperation                 currentCheckpoint;
342                                                                // last checkpoint successfully taken
343                                                                // 
344                                                                // MT - only changed or access in recovery or
345                                                                // checkpoint, both are single thread access
346 
347        long                                         checkpointInstant;
348                                                                // log instant of te curerntCheckpoint
349 
350        private DaemonService         checkpointDaemon;        // the background worker thread who is going to
351                                                                // do checkpoints for this log factory.
352 
353        private        int                        myClientNumber;        
354                                                                // use this number to talk to checkpoint Daemon
355 
356        private volatile boolean checkpointDaemonCalled; 
357                                                                // checkpoint Daemon called already - it is not
358                                                                // important that this value is correct, the
359                                                                // daemon just need to be called once in a
360                                                                // while.  Deamon can handle multiple posts.
361 
362        private long logWrittenFromLastCheckPoint = 0;
363                                    // keeps track of the amout of log written between checkpoints
364        private RawStoreFactory rawStoreFactory; 
365                                                                // use this only after recovery is finished
366 
367        protected DataFactory dataFactory;
368                                                                // use this only after revocery is finished
369 
370        protected boolean        ReadOnlyDB;        // true if this db is read only, i.e, cannot
371                                                                // append log records
372 
373 
374        // DEBUG DEBUG - do not truncate log files
375        private boolean keepAllLogs;
376 
377        // if database is encrypted, the content of the log files are encrypted
378        private boolean databaseEncrypted; 
379 
380        // the following booleans are used to put the log factory into various
381        // states
382        private boolean                         recoveryNeeded = true; // log needs to be recovered
383        private boolean                         inCheckpoint = false;         // in the middle of a checkpoint
384        private boolean                         inRedo = false;        // in the middle of redo loop
385        private boolean          inLogSwitch = false;
386 
387        // make sure we don't do anything after the log factory has been stopped
388        private boolean                         stopped = false;
389 
390        // if log is to go to another device, this variable is set.  If null, then
391        // log goes to the log subdirectory underneath the data directory
392        String logDevice;
393 
394    // disable syncing of log file when running in derby.system.durability=test
395    private boolean logNotSynced = false;
396 
397        private volatile boolean logArchived = false;
398        private boolean logSwitchRequired = false;
399 
400        /** DEBUG test only */
401        int test_logWritten = 0;
402        int test_numRecordToFillLog = -1;
403        private int mon_flushCalls;
404        private int mon_syncCalls;
405        private int mon_numLogFlushWaits;
406        private boolean mon_LogSyncStatistics;
407        private int mon_numBytesToLog;
408 
409 
410        /**
411                If not null then something is corrupt in the raw store and this represents the original error.
412        */
413        protected volatile StandardException corrupt;
414 
415        /**
416                If frozen, don't allow anything on disk to change.
417         */
418        private boolean isFrozen;
419 
420        /**
421          Product Version information. Invarient after boot.
422          */
423        ProductVersionHolder jbmsVersion;
424 
425        /**
426                On disk database version information. When running in soft upgrade this version
427                may be different to jbmsVersion.
428        */
429        private int onDiskMajorVersion;
430        private int onDiskMinorVersion;
431        private boolean onDiskBeta;
432        
433        private CRC32 checksum = new CRC32(); // holder for the checksum
434 
435         
436        /**
437         * Note: Why logging system support file sync and write sync ?
438         * Note : The reason to support file and write sync of logs is 
439         * there was no support to do write sync until jdk1.4 and then
440         * there was write sync jvm bug in jdk1.4.1, only in jdk1.4.2 write 
441         * sync(rws mode)  mechanism can be used corretly.
442         * Default in JVMS >= jdk1.4.2 is write sync(see the boot method for jvm checks).
443         *
444         * Write sync mechanism support is added  for performance reasons. 
445         * On commits, logging system has to make sure the log for committed
446         * transaction is on disk. With out write  sync , log is written to the 
447         * disk and then fsync() is used on commits to make log is written to the 
448         * disk for sure. On most of the OS , fsync() calls are expensive. 
449         * On heavey commit oriented systems , file sync make the system run slow. 
450         * This problem is solved by using write sync on preallocated log file. 
451         * write sync is much faster than doing write and file sync to a file. 
452         * File should be preallocated for write syncs to perform better than
453         * the file sync method. Whenever a new log file is created, 
454         * logSwitchInterval size is preallocated by writing zeros after file after the header. 
455         */
456 
457        /*If set to true , write sync will be used to do log write other file 
458         * level sync is used.
459         */
460        private boolean isWriteSynced = false;
461 
462    
463    // log file that is yet to be copied to backup, updates to this variable 
464    // needs to visible  checkpoint thread. 
465        private volatile long logFileToBackup ; 
466    // It is set to true when  online backup is in progress,  updates to 
467    // this variable needs to visible to checkpoint thread. 
468    private volatile boolean backupInProgress = false; 
469   
470 
471        /**
472                MT- not needed for constructor
473        */
474        public LogToFile() {
475                keepAllLogs = PropertyUtil.getSystemBoolean(RawStoreFactory.KEEP_TRANSACTION_LOG);
476                
477 
478                if (Performance.MEASURE)
479                        mon_LogSyncStatistics = PropertyUtil.getSystemBoolean(LOG_SYNC_STATISTICS);
480        }
481 
482        /*
483        ** Methods of Corruptable
484        */
485 
486        /**
487                Once the log factory is makred as corrupt then the raw sto
488        */
489        public StandardException markCorrupt(StandardException originalError) {
490 
491                boolean firsttime = false;
492 
493                synchronized (this) 
494                {
495                        if (corrupt == null && originalError != null)
496                        {
497                                corrupt = originalError;
498                                firsttime = true;
499                        }
500                }
501 
502                // only print the first error
503                if (corrupt == originalError)
504                        logErrMsg(corrupt);
505 
506 
507                // this is the first time someone detects error, shutdown the
508                // system as much as possible without further damaging it
509                if (firsttime)
510                {
511                        synchronized(this)
512                        {
513                                stopped = true;
514 
515                                if (logOut != null)
516                                {
517                                        try
518                                        {
519                                                logOut.corrupt(); // get rid of open file descriptor
520                                        }
521                                        catch (IOException ioe)
522                                        { 
523                                                // don't worry about it, just trying to clean up 
524                                        }
525                                }
526 
527                                // NullPointerException is preferred over corrupting the database
528                                logOut = null;
529                        }
530 
531                        if (dataFactory != null)
532                                dataFactory.markCorrupt(null);
533 
534                }
535 
536                return originalError;
537        }
538 
539        private void checkCorrupt() throws StandardException
540        {
541                synchronized (this) 
542        {
543                        if (corrupt != null)
544            {
545                                throw StandardException.newException(
546                        SQLState.LOG_STORE_CORRUPT, corrupt);
547            }
548                }
549        }
550 
551        /*
552        ** Methods of LogFactory
553        */
554 
555        /**
556                MT- not needed
557        */
558        public Logger getLogger() {
559 
560                if (ReadOnlyDB)
561                        return null;
562                else
563                        return new FileLogger(this);
564        }
565 
566        /**
567                Recover the rawStore to a consistent state using the log.
568 
569                <P>
570                In this implementation, the log is a stream of log records stored in
571                one or more flat files.  Recovery is done in 2 passes: redo and undo.
572                <BR> <B>Redo pass</B>
573                <BR> In the redo pass, reconstruct the state of the rawstore by
574                repeating exactly what happened before as recorded in the log.
575                <BR><B>Undo pass</B>
576                <BR> In the undo pass, all incomplete transactions are rolled back in
577                the order from the most recently started to the oldest.
578 
579                <P>MT - synchronization provided by caller - RawStore boot.
580                This method is guaranteed to be the only method being called and can
581                assume single thread access on all fields.
582 
583                @see Loggable#needsRedo
584                @see FileLogger#redo
585 
586                @exception StandardException Standard Cloudscape error policy
587        */
588        public void recover(
589    RawStoreFactory     rsf, 
590    DataFactory         df, 
591    TransactionFactory  tf)
592                 throws StandardException
593        {
594                if (SanityManager.DEBUG)
595                {
596                        SanityManager.ASSERT(rsf != null, "raw store factory == null");
597                        SanityManager.ASSERT(df != null,  "data factory == null");
598                }
599 
600                checkCorrupt();
601 
602                rawStoreFactory = rsf;
603                dataFactory     = df;
604                
605                // initialize the log writer only after the rawstorefactory is available, 
606                // log writer requires encryption block size info from rawstore factory 
607                // to encrypt checksum log records. 
608                if (firstLog != null) 
609                        logOut = new LogAccessFile(this, firstLog, logBufferSize);
610 
611                // we don't want to set ReadOnlyDB before recovery has a chance to look
612                // at the latest checkpoint and determine that the database is shutdown
613                // cleanly.  If the medium is read only but there are logs that need
614                // to be redone or in flight transactions, we are hosed.  The logs that
615        // are redone will leave dirty pages in the cache.
616 
617                if (recoveryNeeded)
618                {
619                        try
620                        {
621                                /////////////////////////////////////////////////////////////
622                                //
623                                // During boot time, the log control file is accessed and
624                                // logFileNumber is determined.  LogOut is not set up.
625                                // LogFileNumber is the log file the latest checkpoint lives in,
626                                // or 1.  It may not be the latest log file (the system may have
627                                // crashed between the time a new log was generated and the
628                                // checkpoint log written), that can only be determined at the
629                                // end of recovery redo.
630                                //
631                                /////////////////////////////////////////////////////////////
632                        
633                                FileLogger logger = (FileLogger)getLogger();
634 
635                                /////////////////////////////////////////////////////////////
636                                //
637                                // try to find the most recent checkpoint 
638                                //
639                                /////////////////////////////////////////////////////////////
640                                if (checkpointInstant != LogCounter.INVALID_LOG_INSTANT)
641                {
642                                        currentCheckpoint = 
643                        findCheckpoint(checkpointInstant, logger);
644                }
645 
646                                // if we are only interested in dumping the log, start from the
647                                // beginning of the first log file
648                                if (SanityManager.DEBUG)
649                                {
650                                        if (SanityManager.DEBUG_ON(DUMP_LOG_ONLY))
651                                        {
652                                                currentCheckpoint = null;
653 
654                                                System.out.println("Dump log only");
655 
656                                                // unless otherwise specified, 1st log file starts at 1
657                                                String beginLogFileNumber =
658                                                        PropertyUtil.getSystemProperty(
659                                DUMP_LOG_FROM_LOG_FILE);
660 
661                                                if (beginLogFileNumber != null)
662                        {
663                                                        logFileNumber = 
664                                Long.valueOf(beginLogFileNumber).longValue();
665                        }
666                                                else
667                        {
668                                                        logFileNumber = 1;
669                        }
670                                        }
671                                }
672 
673                                if (SanityManager.DEBUG)
674                                {
675                                        if (SanityManager.DEBUG_ON("setCheckpoint"))
676                                        {
677                                                currentCheckpoint = null;
678 
679                                                System.out.println("Set Checkpoint.");
680 
681                                                // unless otherwise specified, 1st log file starts at 1
682                                                String checkpointStartLogStr =
683                                                        PropertyUtil.getSystemProperty(
684                                "derby.storage.checkpointStartLog");
685 
686                                                String checkpointStartOffsetStr =
687                                                        PropertyUtil.getSystemProperty(
688                                "derby.storage.checkpointStartOffset");
689 
690 
691                                                if ((checkpointStartLogStr != null) && 
692                            (checkpointStartOffsetStr != null))
693                        {
694                                                        checkpointInstant = 
695                                LogCounter.makeLogInstantAsLong(
696                                    Long.valueOf(checkpointStartLogStr).longValue(),
697                                    Long.valueOf(checkpointStartOffsetStr).longValue());
698                        }
699                        else
700                        {
701                            SanityManager.THROWASSERT(
702                                "must set derby.storage.checkpointStartLog and derby.storage.checkpointStartOffset, if setting setCheckpoint.");
703                        }
704 
705                        currentCheckpoint = 
706                            findCheckpoint(checkpointInstant, logger);
707                                        }
708                                }
709 
710                                long redoLWM     = LogCounter.INVALID_LOG_INSTANT;
711                                long undoLWM     = LogCounter.INVALID_LOG_INSTANT;
712                                long ttabInstant = LogCounter.INVALID_LOG_INSTANT;
713 
714                                StreamLogScan redoScan = null;
715                                if (currentCheckpoint != null)
716                                {        
717                                        Formatable transactionTable = null;
718 
719                                        // RESOLVE: sku 
720                                        // currentCheckpoint.getTransactionTable();
721 
722                                        // need to set the transaction table before the undo
723                                        tf.useTransactionTable(transactionTable);
724 
725                                        redoLWM = currentCheckpoint.redoLWM();
726                                        undoLWM = currentCheckpoint.undoLWM();
727 
728                                        if (transactionTable != null)
729                                                ttabInstant = checkpointInstant;
730 
731                                        if (SanityManager.DEBUG)
732                                        {
733                                                if (SanityManager.DEBUG_ON(DBG_FLAG))
734                                                {
735                                                        SanityManager.DEBUG(DBG_FLAG, 
736                                "Found checkpoint at " +
737                                LogCounter.toDebugString(checkpointInstant) + 
738                                " " + currentCheckpoint.toString());
739                                                }
740                                        }
741 
742                                        firstLogFileNumber = LogCounter.getLogFileNumber(redoLWM);
743 
744                                        // figure out where the first interesting log file is.
745                                        if (LogCounter.getLogFileNumber(undoLWM) < 
746                            firstLogFileNumber)
747                    {
748                                                firstLogFileNumber = 
749                            LogCounter.getLogFileNumber(undoLWM);
750                    }
751 
752 
753                                        // if the checkpoint record doesn't have a transaction
754                                        // table, we need to rebuild it by scanning the log from
755                                        // the undoLWM.  If it does have a transaction table, we
756                                        // only need to scan the log from the redoLWM
757 
758                                        redoScan = (StreamLogScan) 
759                        openForwardsScan(undoLWM, (LogInstant)null);
760 
761                                }
762                                else
763                                {
764                                        // no checkpoint
765                                        tf.useTransactionTable((Formatable)null);
766 
767                                        long start = 
768                                                LogCounter.makeLogInstantAsLong(
769                            logFileNumber, LOG_FILE_HEADER_SIZE);
770 
771                                        // no checkpoint, start redo from the beginning of the 
772                    // file - assume this is the first log file
773                                        firstLogFileNumber = logFileNumber;
774 
775                                        redoScan = (StreamLogScan) 
776                        openForwardsScan(start, (LogInstant)null);
777                                }
778 
779                                // open a transaction that is used for redo and rollback
780                                RawTransaction recoveryTransaction =
781                    tf.startTransaction(
782                        rsf,
783                        ContextService.getFactory().getCurrentContextManager(),
784                        AccessFactoryGlobals.USER_TRANS_NAME);
785 
786                                // make this transaction aware that it is a recovery transaction
787                                // and don't spew forth post commit work while replaying the log
788                                recoveryTransaction.recoveryTransaction();
789 
790                                /////////////////////////////////////////////////////////////
791                                //
792                                //  Redo loop - in FileLogger
793                                //
794                                /////////////////////////////////////////////////////////////
795 
796                                // 
797                                // set log factory state to inRedo so that if redo caused any
798                                // dirty page to be written from the cache, it won't flush the
799                                // log since the end of the log has not been determined and we
800                                // know the log record that caused the page to change has
801                                // already been written to the log.  We need the page write to
802                                // go thru the log factory because if the redo has a problem,
803                                // the log factory is corrupt and the only way we know not to
804                                // write out the page in a checkpoint is if it check with the
805                                // log factory, and that is done via a flush - we use the WAL
806                                // protocol to stop corrupt pages from writing to the disk.
807                                //
808                                inRedo = true;        
809 
810                                long logEnd = 
811                    logger.redo(
812                        recoveryTransaction, tf, redoScan, redoLWM, 
813                        ttabInstant);
814 
815                                inRedo = false;
816                                
817 
818                                
819                                // if we are only interested in dumping the log, don't alter
820                                // the database and prevent anyone from using the log
821                                if (SanityManager.DEBUG)
822                                {
823                                        if (SanityManager.DEBUG_ON(LogToFile.DUMP_LOG_ONLY))
824                                        {
825                                                Monitor.logMessage("_____________________________________________________");
826                                                Monitor.logMessage("\n\t\t Log dump finished");
827                                                Monitor.logMessage("_____________________________________________________");
828                        // just in case, it has not been set anyway
829                                                logOut = null; 
830 
831                                                return;
832                                        }
833                                }
834 
835 
836                                /////////////////////////////////////////////////////////////
837                                //
838                                // determine where the log ends
839                                //
840                                /////////////////////////////////////////////////////////////
841                                StorageRandomAccessFile theLog = null;
842 
843 
844                                // if logend == LogCounter.INVALID_LOG_SCAN, that means there 
845                // is no log record in the log - most likely it is corrupted in
846                // some way ...
847                                if (logEnd == LogCounter.INVALID_LOG_INSTANT)
848                                {
849                                        Monitor.logTextMessage(MessageId.LOG_LOG_NOT_FOUND);
850 
851                                        StorageFile logFile = getLogFileName(logFileNumber);
852 
853                    if (privExists(logFile))
854                                        {
855                                                // if we can delete this strange corrupted file, do so,
856                                                // otherwise, skip it
857                        if (!privDelete(logFile))
858                                                {
859                                                        logFile = getLogFileName(++logFileNumber);
860                                                }
861                                        }
862 
863                                        try
864                                        {
865                        theLog =   privRandomAccessFile(logFile, "rw");
866                                        }
867                                        catch (IOException ioe)
868                                        {
869                                                theLog = null;
870                                        }
871 
872                    if (theLog == null || !privCanWrite(logFile))
873                                        {
874                                                if (theLog != null)
875                                                        theLog.close();
876 
877                                                theLog = null;
878 
879                                                ReadOnlyDB = true;
880                                        }
881                                        else
882                                        {
883                                                try
884                                                {
885                                                        // no previous log file or previous log position
886                                                        if (!initLogFile(
887                                    theLog, logFileNumber,
888                                    LogCounter.INVALID_LOG_INSTANT))
889                            {
890                                                                throw markCorrupt(
891                                    StandardException.newException(
892                                        SQLState.LOG_SEGMENT_NOT_EXIST,
893                                        logFile.getPath()));
894                            }
895                                                }
896                                                catch (IOException ioe)
897                                                {
898                                                        throw markCorrupt(
899                                StandardException.newException(
900                                    SQLState.LOG_IO_ERROR, ioe));
901                                                }
902 
903                        // successfully init'd the log file - set up markers,
904                        // and position at the end of the log.
905                                                endPosition = theLog.getFilePointer();
906                                                lastFlush   = endPosition;
907                                                
908                                                //if write sync is true , prellocate the log file
909                                                //and reopen the file in rws mode.
910                                                if(isWriteSynced)
911                                                {
912                                                        //extend the file by wring zeros to it
913                                                        preAllocateNewLogFile(theLog);
914                                                        theLog.close();
915                                                        theLog = openLogFileInWriteMode(logFile);
916                                                        //postion the log at the current end postion
917                                                        theLog.seek(endPosition);
918                                                }
919                                                
920                                                if (SanityManager.DEBUG)
921                                                {
922                                                        SanityManager.ASSERT(
923                                endPosition == LOG_FILE_HEADER_SIZE,
924                                "empty log file has wrong size");
925                                                }
926                                                
927                                                //because we already incrementing the log number
928                                                //here, no special log switch required for
929                                                //backup recoveries.
930                                                logSwitchRequired = false;
931                                        }
932                                }
933                                else
934                                {
935                                        // logEnd is the instant of the next log record in the log
936                                        // it is used to determine the last known good position of
937                                        // the log
938                                        logFileNumber = LogCounter.getLogFileNumber(logEnd);
939 
940                                        ReadOnlyDB = df.isReadOnly();
941 
942                                        StorageFile logFile = getLogFileName(logFileNumber);
943 
944                                        if (!ReadOnlyDB)
945                                        {
946                                                // if datafactory doesn't think it is readonly, we can
947                                                // do some futher test of our own
948                                                try
949                                                {
950                                                        if(isWriteSynced)
951                                                                theLog = openLogFileInWriteMode(logFile);
952                                                        else
953                                                                theLog = privRandomAccessFile(logFile, "rw");
954                                                }
955                                                catch (IOException ioe)
956                                                {
957                                                        theLog = null;
958                                                }
959                        if (theLog == null || !privCanWrite(logFile))
960                                                {
961                                                        if (theLog != null)
962                                                                theLog.close();
963                                                        theLog = null;
964 
965                                                        ReadOnlyDB = true;
966                                                }
967                                        }
968 
969                                        if (!ReadOnlyDB)
970                                        {
971                                                endPosition = LogCounter.getLogFilePosition(logEnd);
972 
973                                                //
974                                                // The end of the log is at endPosition.  Which is where
975                                                // the next log should be appending.
976                                                //
977                                                // if the last log record ends before the end of the
978                        // log file, then this log file has a fuzzy end.
979                        // Zap all the bytes to between endPosition to EOF to 0.
980                                                //
981                                                // the end log marker is 4 bytes (of zeros)
982                                                //
983                                                // if endPosition + 4 == logOut.length, we have a
984                        // properly terminated log file
985                                                //
986                                                // if endPosition + 4 is > logOut.length, there are 0,
987                        // 1, 2, or 3 bytes of 'fuzz' at the end of the log. We
988                        // can ignore that because it is guaranteed to be
989                        // overwritten by the next log record.
990                                                //
991                                                // if endPosition + 4 is < logOut.length, we have a
992                        // partial log record at the end of the log.
993                                                //
994                                                // We need to overwrite all of the incomplete log
995                        // record, because if we start logging but cannot
996                        // 'consume' all the bad log, then the log will truly
997                        // be corrupted if the next 4 bytes (the length of the
998                        // log record) after that is small enough that the next
999                        // time the database is recovered, it will be
1000                        // interpreted that the whole log record is in the log
1001                        // and will try to objectify, only to get classNotFound
1002                        // error or worse.
1003                                                //
1004 
1005                                                //find out if log had incomplete log records at the end.
1006                                                if (redoScan.isLogEndFuzzy())
1007                                                {
1008                                                        theLog.seek(endPosition);
1009                                                        long eof = theLog.length();
1010 
1011                                                        Monitor.logTextMessage(MessageId.LOG_INCOMPLETE_LOG_RECORD,
1012                                                                logFile, new Long(endPosition), new Long(eof));
1013 
1014                                                        /* Write zeros from incomplete log record to end of file */
1015                                                        long nWrites = (eof - endPosition)/logBufferSize;
1016                                                        int rBytes = (int)((eof - endPosition) % logBufferSize);
1017                                                        byte zeroBuf[]= new byte[logBufferSize];
1018                                                        
1019                                                        //write the zeros to file
1020                                                        while(nWrites-- > 0)
1021                                                                theLog.write(zeroBuf);
1022                                                        if(rBytes !=0)
1023                                                                theLog.write(zeroBuf, 0, rBytes);
1024                                                        
1025                                                        if(!isWriteSynced)
1026                                                                syncFile(theLog);
1027                                                }
1028 
1029                                                if (SanityManager.DEBUG)
1030                                                {
1031                                                        if (theLog.length() != endPosition)
1032                                                        {
1033                                                                SanityManager.ASSERT(
1034                                    theLog.length() > endPosition,
1035                                    "log end > log file length, bad scan");
1036                                                        }
1037                                                }
1038 
1039                                                // set the log to the true end position,
1040                        // and not the end of the file
1041 
1042                                                lastFlush = endPosition;
1043                                                theLog.seek(endPosition);
1044                                        }
1045                                }
1046 
1047                                if (theLog != null)
1048                                        logOut = new LogAccessFile(this, theLog, logBufferSize);
1049                                
1050                                if(logSwitchRequired)
1051                                        switchLogFile();
1052 
1053 
1054                                boolean noInFlightTransactions = tf.noActiveUpdateTransaction();
1055 
1056                                if (ReadOnlyDB)
1057                                {
1058                                        // in the unlikely event that someone detects we are
1059                                        // dealing with a read only db, check to make sure the
1060                                        // database is quiesce when it was copied with no unflushed
1061                                        // dirty buffer
1062                                        if (!noInFlightTransactions)
1063                    {
1064                                                throw StandardException.newException(
1065                                SQLState.LOG_READ_ONLY_DB_NEEDS_UNDO);
1066                    }
1067                                }
1068 
1069                                /////////////////////////////////////////////////////////////
1070                                //
1071                                // Undo loop - in transaction factory.  It just gets one
1072                                // transaction at a time from the transaction table and calls
1073                                // undo, no different from runtime.
1074                                //
1075                                /////////////////////////////////////////////////////////////
1076 
1077                if (SanityManager.DEBUG)
1078                {
1079                    if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
1080                        SanityManager.DEBUG(LogToFile.DBG_FLAG,
1081                            "About to call undo(), transaction table =" +
1082                            tf.getTransactionTable());
1083                }
1084 
1085                                if (!noInFlightTransactions)
1086                                {
1087                                        if (SanityManager.DEBUG)
1088                                        {
1089                                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
1090                                                        SanityManager.DEBUG(LogToFile.DBG_FLAG,
1091                                "In recovery undo, rollback inflight transactions");
1092                                        }
1093 
1094                                        tf.rollbackAllTransactions(recoveryTransaction, rsf);
1095 
1096                                        if (SanityManager.DEBUG)
1097                                        {
1098                                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
1099                                                        SanityManager.DEBUG(
1100                                LogToFile.DBG_FLAG, "finish recovery undo,");
1101                                        }
1102                                }
1103                                else
1104                                {
1105                                        if (SanityManager.DEBUG)
1106                                        {
1107                                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
1108                                                        SanityManager.DEBUG(LogToFile.DBG_FLAG,
1109                                "No in flight transaction, no recovery undo work");
1110                                        }
1111                                }
1112 
1113                                /////////////////////////////////////////////////////////////
1114                                //
1115                                // XA prepared xact loop - in transaction factory.  At this
1116                // point only prepared transactions should be left in the
1117                // transaction table, all others should have been aborted or
1118                // committed and removed from the transaction table.  It just
1119                // gets one transaction at a time from the transaction table,
1120                // creates a real context and transaction, reclaims locks,
1121                // and leaves the new xact in the transaction table.
1122                                //
1123                                /////////////////////////////////////////////////////////////
1124 
1125                if (SanityManager.DEBUG)
1126                {
1127                    if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
1128                        SanityManager.DEBUG(LogToFile.DBG_FLAG,
1129                            "About to call rePrepare(), transaction table =" +
1130                            tf.getTransactionTable());
1131                }
1132 
1133                tf.handlePreparedXacts(rsf);
1134 
1135                if (SanityManager.DEBUG)
1136                {
1137                    if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
1138                        SanityManager.DEBUG(LogToFile.DBG_FLAG,
1139                            "Finished rePrepare(), transaction table =" +
1140                            tf.getTransactionTable());
1141                }
1142 
1143                                /////////////////////////////////////////////////////////////
1144                                //
1145                                // End of recovery.
1146                                //
1147                                /////////////////////////////////////////////////////////////
1148 
1149                                // recovery is finished.  Close the transaction
1150                                recoveryTransaction.close();
1151 
1152 
1153                                // notify the dataFactory that recovery is completed,
1154                                // but before the checkpoint is written.
1155                                dataFactory.postRecovery();
1156 
1157 
1158                                //////////////////////////////////////////////////////////////
1159                                // set the transaction factory short id, we have seen all the
1160                                // trasactions in the log, and at the minimum, the checkpoint
1161                                // transaction will be there.  Set the shortId to the next
1162                                // value.
1163                                //////////////////////////////////////////////////////////////
1164                                tf.resetTranId();
1165 
1166                                // do a checkpoint (will flush the log) if there is any rollback
1167                                // if can't checkpoint for some reasons, flush log and carry on
1168                                if (!ReadOnlyDB)
1169                                {
1170                                        boolean needCheckpoint = true;
1171 
1172                                        // if we can figure out there there is very little in the
1173                                        // log (less than 1000 bytes), we haven't done any
1174                    // rollbacks, then don't checkpoint. Otherwise checkpoint.
1175                                        if (currentCheckpoint != null && noInFlightTransactions &&
1176                                                redoLWM != LogCounter.INVALID_LOG_INSTANT &&
1177                                                undoLWM != LogCounter.INVALID_LOG_INSTANT)
1178                                        {
1179                                                if ((logFileNumber == LogCounter.getLogFileNumber(redoLWM))
1180                                                        && (logFileNumber == LogCounter.getLogFileNumber(undoLWM))
1181                                                        && (endPosition < (LogCounter.getLogFilePosition(redoLWM) + 1000)))
1182                                                        needCheckpoint = false;
1183                                        }
1184 
1185                                                if (needCheckpoint && !checkpoint(rsf, df, tf, false))
1186                                                        flush(logFileNumber, endPosition);
1187                                }
1188 
1189                                logger.close();
1190 
1191                                recoveryNeeded = false;
1192                        }
1193                        catch (IOException ioe)
1194                        {
1195                                if (SanityManager.DEBUG)
1196                                        ioe.printStackTrace();
1197 
1198                                throw markCorrupt(
1199                    StandardException.newException(SQLState.LOG_IO_ERROR, ioe));
1200                        }
1201                        catch (ClassNotFoundException cnfe)
1202                        {
1203                                throw markCorrupt(
1204                    StandardException.newException(
1205                        SQLState.LOG_CORRUPTED, cnfe));
1206                        }
1207                        catch (StandardException se)
1208                        {
1209                                throw markCorrupt(se);
1210                        }
1211                        catch (Throwable th)
1212                        {
1213                                if (SanityManager.DEBUG)
1214                {
1215                    SanityManager.showTrace(th);
1216                                        th.printStackTrace();
1217                }
1218 
1219                                throw markCorrupt(
1220                    StandardException.newException(
1221                        SQLState.LOG_RECOVERY_FAILED, th));
1222                        }
1223                }
1224                else
1225                {
1226 
1227                        tf.useTransactionTable((Formatable)null);
1228 
1229                        // set the transaction factory short id
1230                        tf.resetTranId();
1231                }
1232 
1233        // done with recovery        
1234        
1235                /////////////////////////////////////////////////////////////
1236                // setup checktpoint daemon
1237                /////////////////////////////////////////////////////////////
1238                checkpointDaemon = rawStoreFactory.getDaemon();
1239                if (checkpointDaemon != null)
1240        {
1241                        myClientNumber =
1242                checkpointDaemon.subscribe(this, true /*onDemandOnly */);
1243        }
1244        }
1245 
1246 
1247        /**
1248                Checkpoint the rawStore.
1249 
1250                <P> MT- Only one checkpoint is to be taking place at any given time.
1251 
1252                <P> The steps of a checkpoint are
1253 
1254                <OL>
1255                <LI> switch to a new log file if possible
1256                <PRE>
1257                        freeze the log (for the transition to a new log file)
1258                                flush current log file
1259                                create and flush the new log file (with file number 1 higher
1260                than the previous log file). The new log file becomes the
1261                current log file.
1262                        unfreeze the log
1263                </PRE>
1264                <LI> start checkpoint transaction
1265                <LI> gather interesting information about the rawStore:
1266                                        the current log instant (redoLWM)
1267                                        the earliest active transaction begin tran log record 
1268                    instant (undoLWM), all the truncation LWM set by clients 
1269                    of raw store (replication)
1270                <LI> clean the buffer cache 
1271                <LI> log the next checkpoint log record, which contains 
1272                                (repPoint, undoLWM, redoLWM) and commit checkpoint transaction.
1273                <LI> synchronously write the control file containing the next checkpoint
1274                                log record log instant
1275                <LI> the new checkpoint becomes the current checkpoint.
1276                                Somewhere near the beginning of each log file should be a
1277                                checkpoint log record (not guarenteed to be there)
1278                <LI> see if the log can be truncated
1279 
1280                <P>
1281                The earliest useful log record is determined by the repPoint and the 
1282        undoLWM, whichever is earlier. 
1283                <P>
1284                Every log file whose log file number is smaller than the earliest 
1285        useful log record's log file number can be deleted.
1286 
1287                <P><PRE>
1288                        Transactions can be at the following states w/r to a checkpoint -
1289                        consider the log as a continous stream and not as series of log 
1290            files for the sake of clarity.  
1291                        |(BT)-------(ET)| marks the begin and end of a transaction.
1292                        .                          checkpoint started
1293                        .       |__undoLWM          |
1294                        .       V                   |___redoLWM
1295                        .                           |___TruncationLWM
1296                        .                           |
1297                        .                           V
1298                        1 |-----------------|
1299                        2       |--------------------------------|
1300                        3           |-------|
1301                        4               |--------------------------------------(end of log)
1302                        5                                       |-^-|
1303                        .                                   Checkpoint Log Record
1304                        ---A--->|<-------B--------->|<-------------C-----------
1305                </PRE>
1306 
1307                <P>
1308                There are only 3 periods of interest : <BR>
1309                        A) before undoLWM,  B) between undo and redo LWM, C) after redoLWM.
1310 
1311                <P>
1312                Transaction 1 started in A and terminates in B.<BR>
1313                        During redo, we should only see log records and endXact from this
1314                        transaction in the first phase (between undoLWM and redoLWM).  No
1315                        beginXact log record for this transaction will be seen.
1316 
1317                <P>
1318                Transaction 2 started in B (right on the undoLWM) and terminated in C.<BR>
1319                        Any transaction that terminates in C must have a beginXact at or 
1320            after undoLWM.  In other words, no transaction can span A, B and C.
1321                        During redo, we will see beginXact, other log records and endXact 
1322            for this transaction.
1323 
1324                <P>
1325                Transaction 3 started in B and ended in B.<BR>
1326                        During redo, we will see beginXact, other log records and endXact 
1327            for this transaction.
1328 
1329                <P>
1330                Transaction 4 begins in B and never ends.<BR>
1331                        During redo, we will see beginXact, other log records.
1332                        In undo, this loser transaction will be rolled back.
1333 
1334                <P>
1335                Transaction 5 is the transaction taking the checkpoint.<BR>
1336                    The checkpoint action started way back in time but the checkpoint
1337                        log record is only written after the buffer cache has been flushed.
1338 
1339                <P>
1340                Note that if any time elapse between taking the undoLWM and the
1341                        redoLWM, then it will create a 4th period of interest.
1342 
1343                @exception StandardException - encounter exception while doing 
1344                                       checkpoint.
1345        */
1346        public boolean checkpoint(RawStoreFactory rsf,
1347                                                          DataFactory df,
1348                                                          TransactionFactory tf, 
1349                                                          boolean wait)
1350                 throws StandardException
1351        {
1352 
1353                // call checkpoint with no pre-started transaction
1354                boolean done = checkpointWithTran(null, rsf, df, tf);
1355 
1356                //above checpoint call will return 'false'  without
1357                //performing the checkpoint if some other  thread is doing checkpoint. 
1358                //In  cases like backup it is necesary to wait for the 
1359                //checkpoint to complete before copying the files. 'wait' flag get passed 
1360                //in as 'true' by  such cases.
1361                //When wait flag is true, we will wait here until the other thread which
1362                //is actually doing the the checkpoint completes.
1363 
1364                if(!done && wait)
1365                {
1366                        synchronized(this)
1367                        {
1368                                //wait until the thread that is doing the checkpoint completes it.
1369                                while(inCheckpoint)
1370                                {
1371                                        try
1372                                        {
1373                                                wait();
1374                                        }        
1375                                        catch (InterruptedException ie)
1376                                        {
1377                                                throw StandardException.interrupt(ie);
1378                                        }        
1379                                }
1380                                done = true;
1381                        }
1382                }
1383 
1384                return done;
1385        }
1386 
1387 
1388        /**
1389                checkpoint with pre-start transaction
1390 
1391                @exception StandardException Cloudscape Standard Error Policy 
1392        */
1393        protected boolean checkpointWithTran(RawTransaction cptran, 
1394                                                           RawStoreFactory rsf,
1395                                                           DataFactory df,
1396                                                           TransactionFactory tf)
1397                 throws StandardException
1398        {
1399                boolean proceed = true;
1400                LogInstant redoLWM;
1401 
1402                // we may be called to stop the database after a bad error, make sure
1403                // logout is set
1404                if (logOut == null)
1405                {
1406                        return false;
1407                }
1408 
1409                long approxLogLength;
1410 
1411                synchronized (this)
1412                {
1413                        // has someone else found a problem in the raw store?  
1414                        if (corrupt != null)
1415            {
1416                                throw StandardException.newException(SQLState.LOG_STORE_CORRUPT, corrupt);
1417            }
1418 
1419                        // if another checkpoint is in progress, don't do anything
1420                        if (inCheckpoint == true)
1421                                proceed = false;
1422                        else
1423                                inCheckpoint = true;
1424 
1425                        approxLogLength = endPosition; // current end position
1426 
1427                        // don't return from inside of a sync block
1428                }
1429 
1430                if (!proceed)
1431                {
1432                        return false;
1433                }
1434 
1435                // needCPtran == true if we are not supplied with a pre-started transaction
1436                boolean needCPTran = (cptran == null);
1437 
1438                if (SanityManager.DEBUG)
1439                {
1440                        if (logSwitchInterval == 0)
1441                        {
1442                                SanityManager.THROWASSERT(
1443                    "switching log file: Approx log length = " + 
1444                    approxLogLength + " logSwitchInterval = 0");
1445                        }
1446                }
1447 
1448 
1449                try
1450                {
1451                        if (approxLogLength > logSwitchInterval)
1452                        {
1453                                switchLogFile();
1454                                //log switch is occuring in conjuction with the 
1455                                //checkpoint, set the amount of log written from last checkpoint to zero.
1456                                logWrittenFromLastCheckPoint = 0;
1457                        }else
1458                        {
1459                                //checkpoint is happening without the log switch,
1460                                //in the middle of a log file. Amount of log written already for
1461                                //the current log file should not be included in caluculation 
1462                                //of when next check point is due. By assigning the negative
1463                                //value of amount of log writtent for this file. Later it will
1464                                //be subtracted when we switch the log file or while calculating whether 
1465                                //we are due a for checkpoint a flush time.
1466                                logWrittenFromLastCheckPoint = -endPosition;
1467                        }
1468 
1469                        if (SanityManager.DEBUG)
1470                        {
1471                                // if this debug flag is set on, just switch log file
1472 
1473                                if (SanityManager.DEBUG_ON(TEST_LOG_SWITCH_LOG))
1474                                        return false;        
1475                        }
1476 
1477 
1478                        // start a checkpoint transaction 
1479                        if (needCPTran)
1480                                cptran = tf.startInternalTransaction(rsf,
1481                                ContextService.getFactory().getCurrentContextManager());
1482 
1483                        /////////////////////////////////////////////////////
1484                        // gather a snapshot of the various interesting points of the log
1485                        /////////////////////////////////////////////////////
1486                        long undoLWM_long;
1487                        long redoLWM_long;
1488 
1489                        synchronized(this)        // we could synchronized on something else, it
1490                                // doesn't matter as long as logAndDo sync on
1491                                // the same thing
1492                        {
1493                                // The redo LWM is the current log instant.  We are going to 
1494                // clean the cache shortly, any log record before this point 
1495                // will not ever need to be redone.
1496                                redoLWM_long = currentInstant();
1497                                redoLWM = new LogCounter(redoLWM_long);
1498 
1499                // The undo LWM is what we need to rollback all transactions.
1500                // Synchronize this with the starting of a new transaction so 
1501                // that the transaction factory can have a consistent view
1502                // See FileLogger.logAndDo
1503 
1504                                LogCounter undoLWM = (LogCounter)(tf.firstUpdateInstant());
1505                                if (undoLWM == null)
1506                                        undoLWM_long = redoLWM_long; // no active transaction 
1507                                else
1508                                        undoLWM_long = undoLWM.getValueAsLong();
1509 
1510                        }
1511 
1512                        /////////////////////////////////////////////////////
1513                        // clean the buffer cache
1514                        /////////////////////////////////////////////////////
1515                        df.checkpoint();
1516 
1517 
1518                        /////////////////////////////////////////////////////
1519                        // write out the checkpoint log record
1520                        /////////////////////////////////////////////////////
1521                
1522                        // send the checkpoint record to the log
1523                        Formatable transactionTable = tf.getTransactionTable();
1524 
1525                        CheckpointOperation nextCheckpoint = 
1526                                new CheckpointOperation(
1527                    redoLWM_long, undoLWM_long, transactionTable);
1528 
1529                        cptran.logAndDo(nextCheckpoint);
1530 
1531                        LogCounter checkpointInstant = 
1532                (LogCounter)(cptran.getLastLogInstant());
1533 
1534                        if (checkpointInstant != null)
1535            {
1536                // since checkpoint is an internal transaction, I need to 
1537                // flush it to make sure it actually goes to the log
1538                                flush(checkpointInstant); 
1539            }
1540                        else
1541            {
1542                                throw StandardException.newException(
1543                        SQLState.LOG_CANNOT_LOG_CHECKPOINT);
1544            }
1545 
1546                        cptran.commit();
1547 
1548                        if (needCPTran)
1549                        {
1550                                cptran.close();        // if we started it, we will close it
1551                                cptran = null;
1552                        }
1553 
1554                        /////////////////////////////////////////////////////
1555                        // write out the log control file which contains the last
1556                        // successful checkpoint log record
1557                        /////////////////////////////////////////////////////
1558 
1559                        if (!writeControlFile(getControlFileName(),
1560                                                                  checkpointInstant.getValueAsLong()))
1561                        {
1562                                throw StandardException.newException(
1563                        SQLState.LOG_CONTROL_FILE, getControlFileName());
1564                        }
1565 
1566                        // next checkpoint becomes the current checkpoint
1567                        currentCheckpoint = nextCheckpoint;
1568 
1569 
1570                        ////////////////////////////////////////////////////
1571                        // see if we can reclaim some log space
1572                        ////////////////////////////////////////////////////
1573 
1574                        if (!logArchived())
1575                        {
1576                                truncateLog(currentCheckpoint);
1577                        }
1578 
1579                        // delete the committted container drop stubs 
1580            // that are no longer required during recovery. 
1581            // If a backup is in progress don't delete the stubs until 
1582            // it is done. Backup needs to copy all the stubs that 
1583            // are needed to recover from the backup checkpoint on restore.
1584            if(!backupInProgress)
1585                df.removeDroppedContainerFileStubs(redoLWM);
1586                
1587                }
1588                catch (IOException ioe)
1589                {
1590                        throw markCorrupt(
1591                    StandardException.newException(SQLState.LOG_IO_ERROR, ioe));
1592                }
1593                finally 
1594                {
1595                        synchronized(this)
1596                        {
1597                                
1598 
1599                                inCheckpoint = false;
1600                                notifyAll();
1601                        }
1602 
1603                        if (cptran != null && needCPTran)
1604                        {
1605                                try 
1606                                {
1607                                        cptran.commit();
1608                                        cptran.close();
1609                                }
1610                                catch (StandardException se)
1611                                {
1612                                        throw markCorrupt(StandardException.newException(
1613                                            SQLState.LOG_CORRUPTED, se));
1614                                }
1615                        }
1616                }
1617 
1618                return true;
1619        }
1620 
1621        /**
1622                Flush all unwritten log record up to the log instance indicated to disk
1623                and sync.
1624                Also check to see if database is frozen or corrupt.
1625 
1626                <P>MT - not needed, wrapper method
1627 
1628                @param where flush log up to here
1629 
1630                @exception StandardException Standard Cloudscape error policy
1631        */
1632        public void flush(LogInstant where) throws StandardException
1633        {
1634                long fileNumber;
1635                long wherePosition;
1636 
1637                if (where == null) {        
1638                        // don't flush, just use this to check if database is frozen or
1639                        // corrupt 
1640                        fileNumber = 0;
1641                        wherePosition = LogCounter.INVALID_LOG_INSTANT;
1642                } else {
1643                        LogCounter whereC = (LogCounter) where;
1644                        fileNumber = whereC.getLogFileNumber();
1645                        wherePosition = whereC.getLogFilePosition();
1646                }
1647                flush(fileNumber, wherePosition);
1648        }
1649 
1650        /**
1651                Flush all unwritten log record to disk and sync.
1652                Also check to see if database is frozen or corrupt.
1653 
1654                <P>MT - not needed, wrapper method
1655 
1656                @exception StandardException Standard Cloudscape error policy
1657        */
1658        public void flushAll() throws StandardException
1659        {
1660                long fnum;
1661                long whereTo;
1662 
1663                synchronized(this)
1664                {
1665                        fnum = logFileNumber;
1666                        whereTo = endPosition;
1667                }
1668 
1669                flush(fnum, whereTo);
1670        }
1671 
1672        /*
1673         * Private methods that helps to implement methods of LogFactory
1674         */
1675 
1676        /**
1677                Verify that we the log file is of the right format and of the right
1678                version and log file number.
1679 
1680                <P>MT - not needed, no global variables used
1681 
1682                @param logFileName the name of the log file
1683                @param number the log file number
1684                @return true if the log file is of the current version and of the
1685                correct format
1686 
1687                @exception StandardException Standard Cloudscape error policy
1688        */
1689        private boolean verifyLogFormat(StorageFile logFileName, long number)
1690                 throws StandardException 
1691        {
1692                boolean ret = false;
1693                try 
1694                {
1695                        StorageRandomAccessFile log = privRandomAccessFile(logFileName, "r");
1696                        ret = verifyLogFormat(log, number);
1697                        log.close();
1698                }
1699                catch (IOException ioe)
1700                {
1701                        
1702                }
1703 
1704                return ret;
1705        }
1706 
1707        /**
1708                Verify that we the log file is of the right format and of the right
1709                version and log file number.  The log file position is set to the
1710                beginning.
1711 
1712                <P>MT - MT-unsafe, caller must synchronize
1713 
1714                @param log the log file
1715                @param number the log file number
1716                @return true if the log file is of the current version and of the
1717                correct format
1718 
1719                @exception StandardException Standard Cloudscape error policy
1720        */
1721        private boolean verifyLogFormat(StorageRandomAccessFile log, long number) 
1722                 throws StandardException
1723        {
1724                try 
1725                {
1726                        log.seek(0);
1727                        int logfid = log.readInt();
1728                        int obsoleteLogVersion = log.readInt(); // this value is useless, for
1729                                                                // backwards compatibility
1730                        long logNumber = log.readLong();
1731 
1732                        if (logfid != fid || logNumber != number)
1733            {
1734                                throw StandardException.newException(
1735                        SQLState.LOG_INCOMPATIBLE_FORMAT, dataDirectory);
1736            }
1737                }
1738                catch (IOException ioe)
1739                {
1740                        throw StandardException.newException(
1741                    SQLState.LOG_CANNOT_VERIFY_LOG_FORMAT, ioe, dataDirectory);
1742                }
1743 
1744                return true;
1745        }
1746 
1747        /**
1748                Initialize the log to the correct format with the given version and
1749                log file number.  The new log file must be empty.  After initializing,
1750                the file is synchronously written to disk.
1751 
1752                <P>MT - synchornization provided by caller
1753 
1754                @param newlog the new log file to be initialized
1755                @param number the log file number
1756                @param prevLogRecordEndInstant the end position of the  previous log record
1757 
1758                @return true if the log file is empty, else false.
1759 
1760                @exception IOException if new log file cannot be accessed or initialized
1761        */
1762 
1763        private boolean initLogFile(StorageRandomAccessFile newlog, long number,
1764                                                                long prevLogRecordEndInstant)
1765                 throws IOException, StandardException
1766        {
1767                if (newlog.length() != 0)
1768                        return false;
1769 
1770                if (SanityManager.DEBUG)
1771                {
1772                        if ( SanityManager.DEBUG_ON(TEST_LOG_FULL))
1773                                testLogFull();
1774                }
1775                if (SanityManager.DEBUG)
1776                {
1777                        if (SanityManager.DEBUG_ON(TEST_SWITCH_LOG_FAIL1))
1778                                throw new IOException("TestLogSwitchFail1");
1779                }
1780 
1781 
1782                newlog.seek(0);
1783 
1784                newlog.writeInt(fid);
1785                newlog.writeInt(OBSOLETE_LOG_VERSION_NUMBER); // for silly backwards compatibility reason
1786                newlog.writeLong(number);
1787                newlog.writeLong(prevLogRecordEndInstant);
1788 
1789                syncFile(newlog);
1790 
1791                return true;
1792        }
1793 
1794        /**
1795                Switch to the next log file if possible.
1796 
1797                <P>MT - log factory is single threaded thru a log file switch, the log
1798                is frozen for the duration of the switch
1799        */
1800        private void switchLogFile() throws StandardException
1801        {
1802                boolean switchedOver = false;
1803 
1804                /////////////////////////////////////////////////////
1805                // Freeze the log for the switch over to a new log file.
1806                // This blocks out any other threads from sending log
1807                // record to the log stream.
1808                // 
1809                // The switching of the log file and checkpoint are really
1810                // independent events, they are tied together just because a
1811                // checkpoint is the natural place to switch the log and vice
1812                // versa.  This could happen before the cache is flushed or
1813                // after the checkpoint log record is written.
1814                /////////////////////////////////////////////////////
1815                synchronized (this)
1816                {
1817 
1818                        // Make sure that this thread of control is guaranteed to complete
1819            // it's work of switching the log file without having to give up
1820            // the semaphore to a backup or another flusher.  Do this by looping
1821            // until we have the semaphore, the log is not being flushed, and
1822            // the log is not frozen for backup.  Track (2985). 
1823                        while(logBeingFlushed | isFrozen)
1824                        {
1825                                try
1826                                {
1827                                        wait();
1828                                }
1829                                catch (InterruptedException ie)
1830                                {
1831                                        throw StandardException.interrupt(ie);
1832                                }        
1833                        }
1834 
1835                        // we have an empty log file here, refuse to switch.
1836                        if (endPosition == LOG_FILE_HEADER_SIZE)
1837                        {
1838                                if (SanityManager.DEBUG)
1839                                {
1840                                        Monitor.logMessage("not switching from an empty log file (" +
1841                                                   logFileNumber + ")");
1842                                }        
1843                                return;
1844                        }
1845 
1846                        // log file isn't being flushed right now and logOut is not being
1847                        // used.
1848                        StorageFile newLogFile = getLogFileName(logFileNumber+1);
1849 
1850                        if (logFileNumber+1 >= maxLogFileNumber)
1851            {
1852                                throw StandardException.newException(
1853                        SQLState.LOG_EXCEED_MAX_LOG_FILE_NUMBER, 
1854                        new Long(maxLogFileNumber)); 
1855            }
1856 
1857                        StorageRandomAccessFile newLog = null;        // the new log file
1858                        try 
1859                        {
1860                                // if the log file exist and cannot be deleted, cannot
1861                                // switch log right now
1862                if (privExists(newLogFile) && !privDelete(newLogFile))
1863                                {
1864                                        logErrMsg(MessageService.getTextMessage(
1865                        MessageId.LOG_NEW_LOGFILE_EXIST,
1866                                            newLogFile.getPath()));
1867                                        return;
1868                                }
1869 
1870                                try
1871                                {
1872                    newLog =   privRandomAccessFile(newLogFile, "rw");
1873                                }
1874                                catch (IOException ioe)
1875                                {
1876                                        newLog = null;
1877                                }
1878 
1879                if (newLog == null || !privCanWrite(newLogFile))
1880                                {
1881                                        if (newLog != null)
1882                                                newLog.close();
1883                                        newLog = null;
1884 
1885                                        return;
1886                                }
1887 
1888                                if (initLogFile(newLog, logFileNumber+1,
1889                                                                LogCounter.makeLogInstantAsLong(logFileNumber, endPosition)))
1890                                {
1891 
1892                                        // New log file init ok, close the old one and
1893                                        // switch over, after this point, need to shutdown the
1894                                        // database if any error crops up
1895                                        switchedOver = true;
1896 
1897                                        // write out an extra 0 at the end to mark the end of the log
1898                                        // file.
1899                                        
1900                                        logOut.writeEndMarker(0);
1901 
1902                                        endPosition += 4;
1903                                        //set that we are in log switch to prevent flusher 
1904                                        //not requesting  to switch log again 
1905                                        inLogSwitch = true; 
1906                                        // flush everything including the int we just wrote
1907                                        flush(logFileNumber, endPosition);
1908                                        
1909                                        
1910                                        // simulate out of log error after the switch over
1911                                        if (SanityManager.DEBUG)
1912                                        {
1913                                                if (SanityManager.DEBUG_ON(TEST_SWITCH_LOG_FAIL2))
1914                                                        throw new IOException("TestLogSwitchFail2");
1915                                        }
1916 
1917 
1918                                        logOut.close();                // close the old log file
1919                                        
1920                                        logWrittenFromLastCheckPoint += endPosition;
1921 
1922                                        endPosition = newLog.getFilePointer();
1923                                        lastFlush = endPosition;
1924                                        
1925                                        if(isWriteSynced)
1926                                        {
1927                                                //extend the file by wring zeros to it
1928                                                preAllocateNewLogFile(newLog);
1929                                                newLog.close();
1930                                                newLog = openLogFileInWriteMode(newLogFile);
1931                                                newLog.seek(endPosition);
1932                                        }
1933 
1934                                        logOut = new LogAccessFile(this, newLog, logBufferSize);
1935                                        newLog = null;
1936 
1937 
1938                                        if (SanityManager.DEBUG)
1939                                        {
1940                                                if (endPosition != LOG_FILE_HEADER_SIZE)
1941                                                        SanityManager.THROWASSERT(
1942                                                                                        "new log file has unexpected size" +
1943                                                                                         + endPosition);
1944                                        }
1945                                        logFileNumber++;
1946 
1947                                        if (SanityManager.DEBUG)
1948                                        {
1949                                                SanityManager.ASSERT(endPosition == LOG_FILE_HEADER_SIZE,
1950                                                                                         "empty log file has wrong size");
1951                                        }
1952 
1953                                }
1954                                else        // something went wrong, delete the half baked file
1955                                {
1956                                        newLog.close();
1957                                        newLog = null;
1958 
1959                                        if (privExists(newLogFile))
1960                                            privDelete(newLogFile);
1961                                        newLogFile = null;
1962 
1963                                        logErrMsg(MessageService.getTextMessage(
1964                        MessageId.LOG_CANNOT_CREATE_NEW,
1965                        newLogFile.getPath()));
1966                                 }
1967 
1968                        }
1969                        catch (IOException ioe)
1970                        {
1971 
1972                                inLogSwitch = false;
1973                                // switching log file is an optional operation and there is no direct user
1974                                // control.  Just sends a warning message to whomever, if any,
1975                                // system adminstrator there may be
1976 
1977                logErrMsg(MessageService.getTextMessage(
1978                    MessageId.LOG_CANNOT_CREATE_NEW_DUETO,
1979                    newLogFile.getPath(),
1980                    ioe.toString()));
1981 
1982                                try
1983                                {
1984                                        if (newLog != null)
1985                                        {
1986                                                newLog.close();
1987                                                newLog = null;
1988                                        }
1989                                }
1990                                catch (IOException ioe2) {}
1991 
1992                if (newLogFile != null && privExists(newLogFile))
1993                                {
1994                    privDelete(newLogFile);
1995                                        newLogFile = null;
1996                                }
1997 
1998                                if (switchedOver)        // error occur after old log file has been closed!
1999                                {
2000                                        logOut = null; // limit any damage
2001                                        throw markCorrupt(
2002                        StandardException.newException(
2003                                SQLState.LOG_IO_ERROR, ioe));
2004                                }
2005                        }
2006                        
2007                        inLogSwitch = false;
2008                }
2009                // unfreezes the log
2010        }
2011 
2012        /**
2013                Flush all unwritten log record up to the log instance indicated to disk
2014                without syncing.
2015 
2016                <P>MT - not needed, wrapper method
2017 
2018                @param wherePosition flush log up to here
2019 
2020                @exception IOException Failed to flush to the log
2021        */
2022        private void flushBuffer(long fileNumber, long wherePosition)
2023                throws IOException, StandardException
2024        {
2025                synchronized (this) {
2026                        if (fileNumber < logFileNumber)        // history
2027                                return;
2028 
2029                        // A log instant indicates the start of a log record
2030                        // but not how long it is. Thus the amount of data in
2031                        // the logOut buffer is irrelevant. We can only
2032                        // not flush the buffer if the real synced flush
2033                        // included this required log instant. This is because
2034                        // we never flush & sync partial log records.
2035 
2036                        if (wherePosition < lastFlush) // already flushed
2037                                return;
2038 
2039                        // We don't update lastFlush here because lastFlush
2040                        // is the last position in the log file that has been
2041                        // flushed *and* synced to disk. Here we only flush.
2042                        // ie. lastFlush should be renamed lastSync.
2043                        //
2044                        // We could have another variable indicating to which
2045                        // point the log has been flushed which this routine
2046                        // could take advantage of. This would only help rollbacks though.
2047 
2048                        logOut.flushLogAccessFile();
2049                }
2050        }
2051        /** Get rid of old and unnecessary log files
2052 
2053                <P> MT- only one truncate log is allowed to be taking place at any
2054                given time.  Synchronized on this.
2055 
2056         */
2057        private void truncateLog(CheckpointOperation checkpoint)
2058        {
2059                long oldFirstLog;
2060                long firstLogNeeded;
2061 
2062                if (keepAllLogs)
2063                        return;
2064                if ((firstLogNeeded = getFirstLogNeeded(checkpoint))==-1)
2065                        return;
2066                
2067                // when  backup is in progress, log files that are yet to
2068        // be copied to the backup should not be deleted,  even 
2069        // if they are not required  for crash recovery.
2070        if(backupInProgress) {
2071            long logFileNeededForBackup = logFileToBackup;
2072            // check if the log file number is yet to be copied 
2073            // to the backup is less than the log file required 
2074            // for crash recovery, if it is then make the first 
2075            // log file that should not be deleted is the log file 
2076            // that is yet to  be copied to the backup.  
2077            if (logFileNeededForBackup < firstLogNeeded)
2078                firstLogNeeded = logFileNeededForBackup;
2079        }
2080 
2081                oldFirstLog = firstLogFileNumber;
2082                firstLogFileNumber = firstLogNeeded;
2083 
2084                while(oldFirstLog < firstLogNeeded)
2085                {
2086                        StorageFile uselessLogFile = null;
2087                        try
2088                        {
2089                                uselessLogFile = getLogFileName(oldFirstLog);
2090                if (privDelete(uselessLogFile))
2091                                {
2092                                        if (SanityManager.DEBUG)
2093                                        {
2094                                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
2095                                                        SanityManager.DEBUG(DBG_FLAG, "truncating useless log file " + uselessLogFile.getPath());
2096                                        }
2097                                }
2098                                else
2099                                {
2100                                        if (SanityManager.DEBUG)
2101                                        {
2102                                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
2103                                                        SanityManager.DEBUG(DBG_FLAG, "Fail to truncate useless log file " + uselessLogFile.getPath());
2104                                        }
2105                                }
2106                        }
2107                        catch (StandardException se)
2108                        {
2109                                if (SanityManager.DEBUG)
2110                                        SanityManager.THROWASSERT("error opening log segment while deleting "
2111                                                                                          + uselessLogFile.getPath(), se);
2112 
2113                                // if insane, just leave it be
2114                        }
2115 
2116                        oldFirstLog++;
2117                }
2118        }
2119 
2120   
2121 
2122        private long getFirstLogNeeded(CheckpointOperation checkpoint){
2123 
2124                long firstLogNeeded;
2125 
2126                // one truncation at a time
2127                synchronized (this)
2128                {
2129                        firstLogNeeded = (checkpoint != null ? LogCounter.getLogFileNumber(checkpoint.undoLWM()) : -1);
2130 
2131                        if (SanityManager.DEBUG)
2132                        {
2133                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
2134                                        SanityManager.DEBUG(DBG_FLAG, "truncatLog: undoLWM firstlog needed " + firstLogNeeded);
2135                        }
2136 
2137                        if (SanityManager.DEBUG)
2138                        {
2139                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
2140                                {
2141                                SanityManager.DEBUG(DBG_FLAG, "truncatLog: checkpoint truncationLWM firstlog needed " + firstLogNeeded);
2142                                SanityManager.DEBUG(DBG_FLAG, "truncatLog: firstLogFileNumber = " + firstLogFileNumber);
2143                                }
2144                        }
2145                }
2146                return firstLogNeeded;
2147        }
2148 
2149 
2150        /**
2151                Carefully write out this value to the control file.
2152        We do safe write of this data by writing the data 
2153        into two files every time we write the control data.
2154        we write checksum at the end of the file, so if by
2155        chance system crashes while writing into the file,
2156        using the checksum we find that the control file
2157        is hosed then we  use the mirror file, which will have
2158        the condrol data written at last check point.
2159 
2160                see comment at beginning of file for log control file format.
2161 
2162                <P> MT- synchronized by caller
2163        */
2164        boolean writeControlFile(StorageFile logControlFileName, long value)
2165                 throws IOException, StandardException
2166        {
2167                StorageRandomAccessFile logControlFile = null;
2168 
2169                ByteArrayOutputStream baos = new ByteArrayOutputStream(64);
2170                DataOutputStream daos = new DataOutputStream(baos);
2171 
2172                daos.writeInt(fid);
2173 
2174                // so that when this db is booted by 1.1x and 1.2x JBMS, a IOException
2175                // stack trace rather than some error message that tells
2176                // the user to delete the database will show up.
2177                daos.writeInt(OBSOLETE_LOG_VERSION_NUMBER);
2178                daos.writeLong(value);
2179 
2180                if (onDiskMajorVersion == 0) {
2181                        onDiskMajorVersion = jbmsVersion.getMajorVersion();
2182                        onDiskMinorVersion = jbmsVersion.getMinorVersion();
2183                        onDiskBeta = jbmsVersion.isBeta();
2184                }
2185 
2186                // previous to 1.3, that's all we wrote.  
2187                // from 1.3 and onward, also write out the JBMSVersion 
2188                daos.writeInt(onDiskMajorVersion);
2189                daos.writeInt(onDiskMinorVersion);
2190 
2191                // For 2.0 beta we added the build number and the isBeta indication.
2192                // (5 bytes from our first spare long)
2193                daos.writeInt(jbmsVersion.getBuildNumberAsInt());
2194 
2195                byte flags = 0;
2196                if (onDiskBeta) 
2197            flags |= IS_BETA_FLAG;
2198        
2199        // When database is booted with derby.system.durability=test,
2200        // this mode does not guarantee that 
2201        // - database will recover 
2202        // - committed transactions will not be lost
2203        // - database will be in a consistent state
2204        // Hence necessary to keep track of this state so we don't 
2205        // waste time resolving issues in such cases.
2206        // wasDBInDurabilityTestModeNoSync has information if database was
2207        // previously booted at any time in this mode
2208        if (logNotSynced || wasDBInDurabilityTestModeNoSync)
2209            flags |= IS_DURABILITY_TESTMODE_NO_SYNC_FLAG;
2210                 daos.writeByte(flags);
2211 
2212                //
2213                // write some spare bytes after 2.0 we have 3 + 2(8) spare bytes.
2214                 long spare = 0;
2215       
2216                daos.writeByte(0);
2217                daos.writeByte(0);
2218        daos.writeByte(0);
2219                daos.writeLong(spare);
2220                daos.flush();
2221                // write the checksum for the control data written
2222                checksum.reset();
2223                checksum.update(baos.toByteArray(), 0, baos.size());
2224                daos.writeLong(checksum.getValue());
2225                daos.flush();
2226 
2227                try
2228                {
2229            checkCorrupt();
2230 
2231                        try
2232                        {
2233                logControlFile = privRandomAccessFile(logControlFileName, "rw");
2234                        }
2235                        catch (IOException ioe)
2236                        {
2237                                logControlFile = null;
2238                                return false;
2239                        }
2240 
2241            if (!privCanWrite(logControlFileName))
2242                                return false;
2243 
2244                        if (SanityManager.DEBUG)
2245                        {
2246                                if (SanityManager.DEBUG_ON(TEST_LOG_FULL))
2247                                        testLogFull();
2248                        }
2249 
2250                        logControlFile.seek(0);
2251                        logControlFile.write(baos.toByteArray());
2252            syncFile(logControlFile);
2253            logControlFile.close();
2254 
2255                        // write the same data to mirror control file
2256                        try
2257                        {
2258                                logControlFile =
2259                    privRandomAccessFile(getMirrorControlFileName(), "rw");
2260                        }
2261                        catch (IOException ioe)
2262                        {
2263                                logControlFile = null;
2264                                return false;
2265                        }
2266 
2267                        logControlFile.seek(0);
2268                        logControlFile.write(baos.toByteArray());
2269            syncFile(logControlFile);
2270 
2271                }
2272                finally
2273                {
2274                        if (logControlFile != null)
2275                                logControlFile.close();
2276                }
2277 
2278                return true;
2279 
2280        }
2281 
2282        /*
2283                Carefully read the content of the control file.
2284 
2285                <P> MT- read only
2286        */
2287        private long readControlFile(StorageFile logControlFileName, Properties startParams)
2288                 throws IOException, StandardException
2289        {
2290                StorageRandomAccessFile logControlFile = null;
2291                ByteArrayInputStream bais = null;
2292        DataInputStream dais = null;
2293                logControlFile =  privRandomAccessFile(logControlFileName, "r");
2294                boolean upgradeNeeded = false;
2295                long value = LogCounter.INVALID_LOG_INSTANT;
2296                long onDiskChecksum = 0;
2297                long controlFilelength = logControlFile.length();
2298                byte barray[] = null;
2299 
2300                try
2301                {
2302                        // The length of the file is less than the minimum in any version
2303            // It is possibly hosed , no point in reading data from this file
2304            // skip reading checksum  control file is before 1.5
2305            if (controlFilelength < 16)
2306                                onDiskChecksum = -1;
2307                        else if (controlFilelength == 16)
2308                        {
2309                                barray = new byte[16];
2310                                logControlFile.readFully(barray);
2311                        }else if (controlFilelength > 16)
2312            {
2313                                barray = new byte[(int) logControlFile.length() - 8];
2314                                logControlFile.readFully(barray);
2315                                onDiskChecksum = logControlFile.readLong();
2316                                if (onDiskChecksum !=0 )
2317                                {
2318                                        checksum.reset();
2319                                        checksum.update(barray, 0, barray.length);
2320                                }
2321                        }
2322 
2323                        if ( onDiskChecksum == checksum.getValue() || onDiskChecksum ==0)
2324                        {
2325 
2326                                bais = new ByteArrayInputStream(barray);
2327                                dais = new DataInputStream(bais);
2328 
2329                                if (dais.readInt() != fid)
2330                    {
2331                        throw StandardException.newException(
2332                                SQLState.LOG_INCOMPATIBLE_FORMAT, dataDirectory);
2333                    }
2334        
2335                                int obsoleteVersion = dais.readInt();
2336                                value = dais.readLong();
2337        
2338                                if (SanityManager.DEBUG)
2339                                {
2340                                        if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
2341                            SanityManager.DEBUG(LogToFile.DBG_FLAG, 
2342                                "log control file ckp instance = " + 
2343                                LogCounter.toDebugString(value));
2344                                }
2345        
2346        
2347                                // from version 1.5 onward, we added an int for storing JBMS
2348                                // version and an int for storing checkpoint interval
2349                                // and log switch interval
2350                                onDiskMajorVersion = dais.readInt();
2351                                onDiskMinorVersion = dais.readInt();
2352                                int dbBuildNumber = dais.readInt();
2353                                int flags = dais.readByte();
2354                                
2355                                // check if the database was booted previously at any time with
2356                // derby.system.durability=test mode
2357                // If yes, then on a boot error we report that this setting is
2358                // probably the cause for the error and also log a warning
2359                // in the derby.log that this mode was set previously
2360                wasDBInDurabilityTestModeNoSync = 
2361                    (flags & IS_DURABILITY_TESTMODE_NO_SYNC_FLAG) != 0;
2362 
2363                if (SanityManager.DEBUG) {
2364                    if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
2365                        SanityManager.DEBUG(LogToFile.DBG_FLAG,
2366                        "log control file, was derby.system.durability set to test = " +
2367                        wasDBInDurabilityTestModeNoSync);
2368                }
2369                    
2370                                
2371                                onDiskBeta = (flags & IS_BETA_FLAG) != 0;
2372                                if (onDiskBeta)
2373                                {
2374                                        // if is beta, can only be booted by exactly the same
2375                                        // version
2376                                        if (!jbmsVersion.isBeta() ||
2377                                                onDiskMajorVersion != jbmsVersion.getMajorVersion() ||
2378                                                onDiskMinorVersion != jbmsVersion.getMinorVersion())
2379                                        {
2380                                                boolean forceBetaUpgrade = false;
2381                                                if (SanityManager.DEBUG)
2382                                                {
2383                                                        // give ourselves an out for this beta check for debugging purposes
2384                                                        if (SanityManager.DEBUG_ON("forceBetaUpgrade"))
2385                                                        {
2386                                                                Monitor.logMessage("WARNING !! : forcing beta upgrade.");
2387                                                                forceBetaUpgrade =true;
2388                                                        }
2389                                                }
2390 
2391                                                if (!forceBetaUpgrade)
2392                            {
2393                                                        throw StandardException.newException(
2394                                    SQLState.LOG_CANNOT_UPGRADE_BETA, 
2395                                    dataDirectory,
2396                                                                ProductVersionHolder.simpleVersionString(onDiskMajorVersion, onDiskMinorVersion, onDiskBeta));
2397                            }
2398                                        }
2399                                }
2400                                        
2401        
2402                                // JBMS_VERSION must be numbered in a way so that it is ever
2403                                // increasing.  We are backwards compatible but not forwards
2404                                // compatible 
2405                                //
2406                                if (onDiskMajorVersion > jbmsVersion.getMajorVersion() ||
2407                                        (onDiskMajorVersion == jbmsVersion.getMajorVersion() &&
2408                                         onDiskMinorVersion > jbmsVersion.getMinorVersion()))
2409                                {
2410                                        // don't need to worry about point release, no format
2411                                        // upgrade is allowed. 
2412                                        throw StandardException.newException(
2413                                SQLState.LOG_INCOMPATIBLE_VERSION, 
2414                                dataDirectory,
2415                                                        ProductVersionHolder.simpleVersionString(onDiskMajorVersion, onDiskMinorVersion, onDiskBeta));
2416                                }
2417 
2418                                // Ensure that upgrade has been requested for a major or minor upgrade
2419                                // maintaince (point) versions should not require an upgrade.
2420                                if ((onDiskMajorVersion != jbmsVersion.getMajorVersion()) ||
2421                                        (onDiskMinorVersion != jbmsVersion.getMinorVersion()))
2422                                {
2423                                        upgradeNeeded = true;
2424                                }
2425                                // if checksum is zeros in  version > 3.5 file is hosed
2426                                // except incase of upgrade from versions <= 3.5
2427                                if (onDiskChecksum == 0 && 
2428                                        (!(onDiskMajorVersion <= 3 && onDiskMinorVersion <=5) ||
2429                                        onDiskMajorVersion == 0))
2430                                        value = LogCounter.INVALID_LOG_INSTANT; 
2431                        }
2432                }
2433                finally
2434                {
2435                        if (logControlFile != null)
2436                                logControlFile.close();
2437                        if (bais != null)
2438                                bais.close();
2439                        if (dais != null)
2440                                dais.close();
2441                }
2442 
2443                if (upgradeNeeded)
2444                {
2445                        if (Monitor.isFullUpgrade(startParams,
2446                                ProductVersionHolder.simpleVersionString(onDiskMajorVersion, onDiskMinorVersion, onDiskBeta))) {
2447 
2448                                onDiskMajorVersion = jbmsVersion.getMajorVersion();
2449                                onDiskMinorVersion = jbmsVersion.getMinorVersion();
2450                                onDiskBeta = jbmsVersion.isBeta();
2451 
2452                                // Write out the new log control file with the new
2453                                // version, the database has been upgraded
2454 
2455                                if (!writeControlFile(logControlFileName, value))
2456                                {
2457                                        throw StandardException.newException(
2458                                                        SQLState.LOG_CONTROL_FILE, logControlFileName);
2459                                }
2460                        }
2461                }
2462 
2463                return value;
2464 
2465        }
2466 
2467 
2468 
2469    /**
2470     * Create the directory where transaction log should go.
2471     * @exception StandardException Standard Error Policy
2472    */
2473        private void createLogDirectory() throws StandardException
2474        {
2475                StorageFile logDir = 
2476            logStorageFactory.newStorageFile(LogFactory.LOG_DIRECTORY_NAME);
2477 
2478        if (privExists(logDir)) {
2479            // make sure log directory is empty.
2480            String[] logfiles = privList(logDir);
2481            if (logfiles != null) {
2482                if(logfiles.length != 0) {
2483                    throw StandardException.newException(
2484                        SQLState.LOG_SEGMENT_EXIST, logDir.getPath());
2485                }
2486            }
2487            
2488        }else {
2489            // create the log directory.
2490            if (!privMkdirs(logDir)) {
2491                throw StandardException.newException(
2492                    SQLState.LOG_SEGMENT_NOT_EXIST, logDir.getPath());
2493            }
2494        }
2495        }
2496 
2497        /*
2498                Return the directory the log should go.
2499 
2500                <P> MT- read only
2501                @exception StandardException Cloudscape Standard Error Policy
2502        */
2503        public StorageFile getLogDirectory() throws StandardException
2504        {
2505                StorageFile logDir = null;
2506 
2507                logDir = logStorageFactory.newStorageFile( LogFactory.LOG_DIRECTORY_NAME);
2508 
2509        if (!privExists(logDir))
2510                {
2511                        throw StandardException.newException(
2512                    SQLState.LOG_SEGMENT_NOT_EXIST, logDir.getPath());
2513                }
2514 
2515                return logDir;
2516        }
2517 
2518        public String getCanonicalLogPath()
2519        {
2520                if (logDevice == null)
2521                        return null;
2522                else
2523                {
2524                        try
2525                        {
2526                                return logStorageFactory.getCanonicalName();
2527                        }
2528                        catch (IOException ioe)
2529                        {
2530                                return null;
2531                                // file not found
2532                        }
2533                }
2534        }
2535 
2536 
2537        /**
2538                Return the control file name 
2539 
2540                <P> MT- read only
2541        */
2542        private StorageFile getControlFileName() throws StandardException
2543        {
2544                return logStorageFactory.newStorageFile( getLogDirectory(), "log.ctrl");
2545        }
2546 
2547        /**
2548                Return the mirror control file name 
2549 
2550                <P> MT- read only
2551        */
2552        private StorageFile getMirrorControlFileName() throws StandardException
2553        {
2554                return logStorageFactory.newStorageFile( getLogDirectory(), "logmirror.ctrl");
2555        }
2556 
2557        /**
2558                Given a log file number, return its file name 
2559 
2560                <P> MT- read only
2561        */
2562        private StorageFile getLogFileName(long filenumber) throws StandardException
2563        {
2564                return logStorageFactory.newStorageFile( getLogDirectory(), "log" + filenumber + ".dat");
2565        }
2566 
2567        /*
2568                Find a checkpoint log record at the checkpointInstant
2569 
2570                <P> MT- read only
2571        */
2572        private CheckpointOperation findCheckpoint(long checkpointInstant, FileLogger logger)
2573                 throws IOException, StandardException, ClassNotFoundException
2574        {
2575                StreamLogScan scan = (StreamLogScan)
2576                        openForwardsScan(checkpointInstant, (LogInstant)null);
2577 
2578                // estimated size of a checkpoint log record, which contains 3 longs
2579                // and assorted other log record overhead
2580                Loggable lop = logger.readLogRecord(scan, 100);
2581                                                                
2582                scan.close();
2583 
2584                if (lop instanceof CheckpointOperation)
2585                        return (CheckpointOperation)lop;
2586                else
2587                        return null;
2588        }
2589 
2590 
2591        /*
2592         * Functions to help the Logger open a log scan on the log.
2593         */
2594 
2595        /**
2596                Scan backward from start position.
2597 
2598                <P> MT- read only
2599 
2600                @exception IOException cannot access the log
2601                @exception StandardException Standard Cloudscape error policy
2602        */
2603        protected LogScan openBackwardsScan(long startAt, LogInstant stopAt)  
2604                 throws IOException, StandardException
2605        {
2606                checkCorrupt();
2607 
2608                // backward from end of log
2609                if (startAt == LogCounter.INVALID_LOG_INSTANT)
2610                        return openBackwardsScan(stopAt);
2611 
2612 
2613                // ensure any buffered data is written to the actual file
2614                flushBuffer(LogCounter.getLogFileNumber(startAt),
2615                                LogCounter.getLogFilePosition(startAt));
2616 
2617                return new Scan(this, startAt, stopAt, Scan.BACKWARD);
2618        }
2619 
2620        /**
2621                Scan backward from end of log.
2622                <P> MT- read only
2623 
2624                @exception IOException cannot access the log
2625                @exception StandardException Standard Cloudscape error policy
2626        */
2627        protected LogScan openBackwardsScan(LogInstant stopAt)
2628                 throws IOException, StandardException
2629        {
2630                checkCorrupt();
2631 
2632                // current instant log instant of the next log record to be
2633                // written out, which is at the end of the log
2634                // ensure any buffered data is written to the actual file
2635                long startAt;
2636                synchronized (this)
2637                {
2638                        // flush the whole buffer to ensure the complete
2639                        // end of log is in the file.
2640                        logOut.flushLogAccessFile();
2641                        startAt = currentInstant();        
2642                }
2643 
2644                return new Scan(this, startAt, stopAt, Scan.BACKWARD_FROM_LOG_END);
2645        }
2646 
2647        /**
2648          @see LogFactory#openFlushedScan
2649          @exception StandardException Ooops.
2650         */
2651        public ScanHandle openFlushedScan(DatabaseInstant start,int groupsIWant)
2652                 throws StandardException
2653        {
2654                return new FlushedScanHandle(this,start,groupsIWant);
2655        }
2656        
2657 
2658 
2659        /**
2660                Scan Forward from start position.
2661 
2662                <P> MT- read only
2663 
2664                @param startAt - if startAt == INVALID_LOG_INSTANT,
2665                        start from the beginning of the log. Otherwise, start scan from startAt.
2666                @param stopAt - if not null, stop at this log instant (inclusive).
2667                        Otherwise, stop at the end of the log
2668 
2669                @exception IOException cannot access the log
2670                @exception StandardException Standard Cloudscape error policy
2671        */
2672        protected LogScan openForwardsScan(long startAt, LogInstant stopAt)  
2673                 throws IOException, StandardException
2674        {
2675                checkCorrupt();
2676 
2677                if (startAt == LogCounter.INVALID_LOG_INSTANT)
2678                {
2679                        startAt = firstLogInstant();
2680                }
2681 
2682                // ensure any buffered data is written to the actual file
2683                if (stopAt != null) {
2684                        LogCounter stopCounter = (LogCounter) stopAt;
2685                        flushBuffer(stopCounter.getLogFileNumber(),
2686                                                stopCounter.getLogFilePosition());
2687                } else {
2688                        synchronized (this) {
2689                                if (logOut != null)
2690                                        // flush to the end of the log
2691                                        logOut.flushLogAccessFile();
2692                        }
2693                }
2694 
2695                return new Scan(this, startAt, stopAt, Scan.FORWARD);
2696        }
2697 
2698        /*
2699         * Methods to help a log scan switch from one log file to the next 
2700         */
2701 
2702        /**
2703                Open a log file and position the file at the beginning.
2704                Used by scan to switch to the next log file
2705 
2706                <P> MT- read only
2707 
2708                @exception StandardException Standard Cloudscape error policy
2709                @exception IOException cannot access the log at the new position.
2710        */
2711        protected StorageRandomAccessFile getLogFileAtBeginning(long filenumber)
2712                 throws IOException, StandardException
2713        {
2714                long instant = LogCounter.makeLogInstantAsLong(filenumber,
2715                                                                                                           LOG_FILE_HEADER_SIZE);
2716                return getLogFileAtPosition(instant);
2717        }
2718 
2719 
2720        /**
2721                Get a read-only handle to the log file positioned at the stated position
2722 
2723                <P> MT- read only
2724 
2725                @return null if file does not exist or of the wrong format
2726                @exception IOException cannot access the log at the new position.
2727                @exception StandardException Standard Cloudscape error policy
2728        */
2729        protected StorageRandomAccessFile getLogFileAtPosition(long logInstant)
2730                 throws IOException, StandardException
2731        {
2732                checkCorrupt();
2733 
2734                long filenum = LogCounter.getLogFileNumber(logInstant);
2735                long filepos = LogCounter.getLogFilePosition(logInstant);
2736 
2737                StorageFile fileName = getLogFileName(filenum);
2738        if (!privExists(fileName))
2739                {
2740                        if (SanityManager.DEBUG)
2741                        {
2742                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
2743                                        SanityManager.DEBUG(LogToFile.DBG_FLAG, fileName.getPath() + " does not exist");
2744                        }
2745 
2746                        return null;
2747                }
2748 
2749 
2750                StorageRandomAccessFile log = null;
2751 
2752                try
2753                {
2754            log = privRandomAccessFile(fileName, "r");
2755 
2756                        // verify that the log file is of the right format
2757                        if (!verifyLogFormat(log, filenum))
2758                        {
2759                                if (SanityManager.DEBUG)
2760                                {
2761                                        if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
2762                                                SanityManager.DEBUG(LogToFile.DBG_FLAG, fileName.getPath() + " format mismatch");
2763                                }
2764 
2765                                log.close();
2766                                log = null;
2767                        }
2768                        else
2769                        {
2770                                log.seek(filepos);
2771                        }
2772                }
2773                catch (IOException ioe)
2774                {
2775                        try
2776                        {
2777                                if (log != null)
2778                                {
2779                                        log.close();
2780                                        log = null;
2781                                }
2782 
2783                                if (SanityManager.DEBUG)
2784                                {
2785                                        SanityManager.THROWASSERT("cannot get to position " + filepos +
2786                                                                                          " for log file " + fileName.getPath(), ioe);
2787                                }
2788                        }
2789                        catch (IOException ioe2)
2790                        {}
2791                        throw ioe;
2792                }
2793 
2794                return log;
2795 
2796        }
2797 
2798        /*
2799        ** Methods of ModuleControl
2800        */
2801 
2802        public boolean canSupport(Properties startParams)
2803        {
2804                String runtimeLogAttributes = startParams.getProperty(LogFactory.RUNTIME_ATTRIBUTES);
2805                if (runtimeLogAttributes != null) {
2806                        if (runtimeLogAttributes.equals(LogFactory.RT_READONLY))
2807                                return false;
2808                }
2809 
2810                return        true;
2811        }
2812 
2813 
2814 
2815 
2816        /**
2817                Boot up the log factory.
2818                <P> MT- caller provide synchronization
2819 
2820                @exception StandardException log factory cannot start up
2821        */
2822        public void        boot(boolean create, Properties startParams) throws StandardException
2823        {
2824                dataDirectory = startParams.getProperty(PersistentService.ROOT);
2825        
2826                logDevice = startParams.getProperty(Attribute.LOG_DEVICE);
2827        if( logDevice != null)
2828        {
2829                        // in case the user specifies logDevice in URL form
2830                        String logDeviceURL = null;
2831                        try {
2832                                URL url = new URL(logDevice);
2833                                logDeviceURL = url.getFile();
2834                        } catch (MalformedURLException ex) {}
2835                        if (logDeviceURL != null)
2836                                logDevice = logDeviceURL;
2837        }
2838 
2839 
2840        if(create) {
2841            getLogStorageFactory();
2842            createLogDirectory();
2843            
2844        } else {
2845            // check if the database is being restored from the backup,
2846            // if it is then restore the logs.
2847            if (!restoreLogs(startParams)) {
2848                // set the log storage factory.
2849                getLogStorageFactory();
2850                if (logDevice != null)
2851                {
2852                    // Make sure we find the log, do not assume 
2853                    // it is OK that the log is not there because 
2854                    // it could be a user typo(like when users edit
2855                    // service.properties to change the log device 
2856                    // while restoring from backups using OS copy.
2857                    StorageFile checklogDir =
2858                        logStorageFactory.newStorageFile( 
2859                                 LogFactory.LOG_DIRECTORY_NAME);
2860                    if (!privExists(checklogDir))
2861                    {
2862                        throw
2863                            StandardException.newException(
2864                            SQLState.LOG_FILE_NOT_FOUND, checklogDir.getPath());
2865 
2866                    }
2867                }
2868            }
2869        }
2870                        
2871                //if user does not set the right value for the log buffer size,
2872                //default value is used instead.
2873                logBufferSize =  PropertyUtil.getSystemInt(org.apache.derby.iapi.reference.Property.LOG_BUFFER_SIZE, 
2874                                                                                                   LOG_BUFFER_SIZE_MIN, 
2875                                                                                                   LOG_BUFFER_SIZE_MAX, 
2876                                                                                                   DEFAULT_LOG_BUFFER_SIZE);
2877                jbmsVersion = Monitor.getMonitor().getEngineVersion();
2878 
2879                
2880                String logArchiveMode = 
2881            startParams.getProperty(Property.LOG_ARCHIVE_MODE);
2882                logArchived = Boolean.valueOf(logArchiveMode).booleanValue();
2883                
2884                //get log factorty properties if any set in derby.properties
2885                getLogFactoryProperties(null);
2886 
2887                /* check if the storage factory supports write sync(rws).  If so, use it unless
2888                 * derby.storage.fileSyncTransactionLog property is set true by user.
2889                 */
2890 
2891                if (logStorageFactory.supportsRws())
2892        {
2893                        //write sync can be used in the jvm that database is running on.
2894                        //disable write sync if derby.storage.fileSyncTransactionLog is true
2895                        isWriteSynced =
2896                                !(PropertyUtil.getSystemBoolean(Property.FILESYNC_TRANSACTION_LOG));
2897        }
2898                else
2899                {
2900                        isWriteSynced = false;
2901                }
2902 
2903 
2904        // If derby.system.durability=test is set,then set flag to 
2905        // disable sync of log records at commit and log file before 
2906        // data page makes it to disk
2907        if (Property.DURABILITY_TESTMODE_NO_SYNC.equalsIgnoreCase(
2908               PropertyUtil.getSystemProperty(Property.DURABILITY_PROPERTY)))
2909        {
2910                    // disable syncing of log.
2911                    logNotSynced = true;
2912                      //if log not being synced;files shouldn't be open in write sync mode
2913                    isWriteSynced = false;        
2914                }
2915        else if (Performance.MEASURE)
2916        {
2917            // development build only feature, must by hand set the 
2918            // Performance.MEASURE variable and rebuild.  Useful during
2919            // development to compare/contrast effect of syncing, release
2920            // users can use the above relaxed durability option to disable
2921            // all syncing.  
2922 
2923            logNotSynced = 
2924                PropertyUtil.getSystemBoolean(
2925                    Property.STORAGE_LOG_NOT_SYNCED);
2926 
2927            if (logNotSynced)
2928            {
2929                isWriteSynced = false;
2930                Monitor.logMessage("Performance.logNotSynced = true");
2931            }
2932        }
2933 
2934                // try to access the log
2935                // if it doesn't exist, create it.
2936                // if it does exist, run recovery
2937 
2938                boolean createNewLog = create;
2939 
2940                if (SanityManager.DEBUG)
2941                        SanityManager.ASSERT(fid != -1, "invalid log format Id");
2942 
2943                checkpointInstant = LogCounter.INVALID_LOG_INSTANT;
2944                try
2945                {
2946                        StorageFile logControlFileName = getControlFileName();
2947 
2948                        StorageFile logFile;
2949 
2950                        if (!createNewLog)
2951                        {
2952                if (privExists(logControlFileName))
2953                                {
2954                                        checkpointInstant = 
2955                        readControlFile(logControlFileName, startParams);
2956 
2957                                        // in case system was running previously with 
2958                    // derby.system.durability=test then print a message 
2959                    // to the derby log
2960                    if (wasDBInDurabilityTestModeNoSync)
2961                    {
2962                        // print message stating that the database was
2963                        // previously atleast at one time running with
2964                        // derby.system.durability=test mode
2965                        Monitor.logMessage(MessageService.getTextMessage(
2966                                                   MessageId.LOG_WAS_IN_DURABILITY_TESTMODE_NO_SYNC,
2967                                                   Property.DURABILITY_PROPERTY,
2968                            Property.DURABILITY_TESTMODE_NO_SYNC));
2969                    }
2970                                                
2971                                        if (checkpointInstant == LogCounter.INVALID_LOG_INSTANT &&
2972                                                                                getMirrorControlFileName().exists())
2973                    {
2974                                                checkpointInstant =
2975                            readControlFile(
2976                                getMirrorControlFileName(), startParams);
2977                    }
2978 
2979                                }
2980                                else if (logDevice != null)
2981                                {
2982                                        // Do not throw this error if logDevice is null because
2983                                        // in a read only configuration, it is acceptable
2984                                        // to not have a log directory.  But clearly, if the
2985                                        // logDevice property is set, then it should be there.
2986                                        throw StandardException.newException(
2987                            SQLState.LOG_FILE_NOT_FOUND,
2988                            logControlFileName.getPath());
2989                                }
2990 
2991                                if (checkpointInstant != LogCounter.INVALID_LOG_INSTANT)
2992                                        logFileNumber = LogCounter.getLogFileNumber(checkpointInstant);
2993                                else
2994                                        logFileNumber = 1;
2995 
2996                                logFile = getLogFileName(logFileNumber);
2997 
2998                                // if log file is not there or if it is of the wrong format, create a
2999                                // brand new log file and do not attempt to recover the database
3000 
3001                if (!privExists(logFile))
3002                                {
3003                                        if (logDevice != null)
3004                    {
3005                        throw StandardException.newException(
3006                                SQLState.LOG_FILE_NOT_FOUND,
3007                                logControlFileName.getPath());
3008                    }
3009 
3010                                        logErrMsg(MessageService.getTextMessage(
3011                        MessageId.LOG_MAYBE_INCONSISTENT,
3012                        logFile.getPath()));
3013 
3014                                        createNewLog = true;
3015                                }
3016                                else if (!verifyLogFormat(logFile, logFileNumber))
3017                                {
3018                                        Monitor.logTextMessage(MessageId.LOG_DELETE_INCOMPATIBLE_FILE, logFile);
3019 
3020                                        // blow away the log file if possible
3021                    if (!privDelete(logFile) && logFileNumber == 1)
3022                    {
3023                        logErrMsgForDurabilityTestModeNoSync();
3024                                                throw StandardException.newException(
3025                            SQLState.LOG_INCOMPATIBLE_FORMAT, dataDirectory);
3026                    }
3027 
3028                                        // If logFileNumber > 1, we are not going to write that 
3029                    // file just yet.  Just leave it be and carry on.  Maybe 
3030                    // when we get there it can be deleted.
3031 
3032                                        createNewLog = true;
3033                                }
3034                        }
3035 
3036                        if (createNewLog)
3037                        {
3038                                // brand new log.  Start from log file number 1.
3039 
3040                                // create or overwrite the log control file with an invalid
3041                                // checkpoint instant since there is no checkpoint yet
3042                                if (writeControlFile(logControlFileName,
3043                                                                         LogCounter.INVALID_LOG_INSTANT))
3044                                {
3045                                        firstLogFileNumber = 1;
3046                                        logFileNumber = 1;
3047                                        if (SanityManager.DEBUG)
3048                                        {
3049                                                if (SanityManager.DEBUG_ON(TEST_MAX_LOGFILE_NUMBER))
3050                                                {
3051                                                        // set the value to be two less than max possible
3052                                                        // log number, test case will perform some ops to 
3053                                                        // hit the max number case.
3054                                                        firstLogFileNumber = 
3055                                LogCounter.MAX_LOGFILE_NUMBER -2;
3056 
3057                                                        logFileNumber = LogCounter.MAX_LOGFILE_NUMBER -2;
3058                                                }
3059                                        }
3060                                        logFile = getLogFileName(logFileNumber);
3061 
3062                    if (privExists(logFile))
3063                                        {
3064                                                // this log file maybe there because the system may have
3065                                                // crashed right after a log switch but did not write
3066                        // out any log record
3067                                                Monitor.logTextMessage(
3068                            MessageId.LOG_DELETE_OLD_FILE, logFile);
3069 
3070                        if (!privDelete(logFile))
3071                        {
3072                            logErrMsgForDurabilityTestModeNoSync();
3073                                                        throw StandardException.newException(
3074                                    SQLState.LOG_INCOMPATIBLE_FORMAT,
3075                                    dataDirectory);
3076                        }
3077                                        }
3078 
3079                                        // don't need to try to delete it, we know it isn't there
3080                    firstLog = privRandomAccessFile(logFile, "rw");
3081 
3082                                        if (!initLogFile(firstLog, logFileNumber, LogCounter.INVALID_LOG_INSTANT))
3083                    {
3084                                                throw StandardException.newException(
3085                            SQLState.LOG_SEGMENT_NOT_EXIST, logFile.getPath());
3086                    }
3087 
3088                                        endPosition = firstLog.getFilePointer();
3089                                        lastFlush = firstLog.getFilePointer();
3090 
3091                    //if write sync is true , prellocate the log file
3092                    //and reopen the file in rws mode.
3093                    if(isWriteSynced)
3094                    {
3095                        //extend the file by wring zeros to it
3096                        preAllocateNewLogFile(firstLog);
3097                        firstLog.close();
3098                        firstLog = openLogFileInWriteMode(logFile);
3099                        //postion the log at the current log end postion
3100                        firstLog.seek(endPosition);
3101                    }
3102 
3103                                        if (SanityManager.DEBUG)
3104                                        {
3105                                                SanityManager.ASSERT(
3106                            endPosition == LOG_FILE_HEADER_SIZE,
3107                            "empty log file has wrong size");
3108                                        }
3109                                }
3110                                else
3111                                {
3112                                        // read only database
3113                                        ReadOnlyDB = true;
3114                                        logOut = null;
3115                                        firstLog = null;
3116                                }
3117 
3118                                recoveryNeeded = false;
3119                        }
3120                        else
3121                        {
3122                                // log file exist, need to run recovery
3123                                recoveryNeeded = true;
3124                        }
3125 
3126                }
3127                catch (IOException ioe)
3128                {
3129                        throw Monitor.exceptionStartingModule(ioe);
3130                }
3131                        
3132                // Number of the log file that can be created in Derby is increased from 
3133                // 2^22 -1 to 2^31 -1 in version 10.1. But if the database is running on
3134                // engines 10.1 or above on a  softupgrade  from versions 10.0 or
3135                // before, the max log file number  that can be created is  
3136                // still limited to 2^22 -1, because users can revert back to older  versions 
3137                // which does not have logic to handle a log file number greater than
3138                // 2^22-1. 
3139 
3140                // set max possible log file number to derby 10.0 limit, if the database is not 
3141                // fully upgraded to or created in version 10.1 or above. 
3142                if (!checkVersion(RawStoreFactory.DERBY_STORE_MAJOR_VERSION_10, 
3143                                                  RawStoreFactory.DERBY_STORE_MINOR_VERSION_1))
3144                        maxLogFileNumber = LogCounter.DERBY_10_0_MAX_LOGFILE_NUMBER;
3145 
3146        } // end of boot
3147 
3148    private void getLogStorageFactory() throws StandardException
3149    {
3150        if( logDevice == null)
3151        {
3152            DataFactory df = (DataFactory) Monitor.findServiceModule( this, DataFactory.MODULE);
3153            logStorageFactory = (WritableStorageFactory) df.getStorageFactory();
3154        }
3155        else
3156        {
3157            try
3158            {
3159                PersistentService ps = Monitor.getMonitor().getServiceType(this);
3160                logStorageFactory = (WritableStorageFactory) ps.getStorageFactoryInstance( false, logDevice, null, null);
3161            }
3162            catch( IOException ioe)
3163            {
3164                if( SanityManager.DEBUG)
3165                    SanityManager.NOTREACHED();
3166                throw StandardException.newException( SQLState.LOG_FILE_NOT_FOUND, ioe, logDevice);
3167            }
3168        }
3169    } // end of getLogStorageFactory
3170  
3171        /**
3172            Stop the log factory
3173                <P> MT- caller provide synchronization
3174                (RESOLVE: this should be called AFTER dataFactory and transFactory are
3175                stopped)
3176        */
3177        public  void stop() {
3178 
3179 
3180                // stop our checkpoint 
3181                if (checkpointDaemon != null) {
3182                        checkpointDaemon.unsubscribe(myClientNumber);
3183                        checkpointDaemon.stop();
3184                }
3185 
3186                synchronized(this)
3187                {
3188                        stopped = true;
3189 
3190                        if (logOut != null) {
3191                                try {
3192                                        logOut.flushLogAccessFile();
3193                                        logOut.close();
3194                                }
3195                                catch (IOException ioe) {}
3196                                catch(StandardException se){}
3197                                logOut = null;
3198                        }
3199                }
3200 
3201          
3202                if (SanityManager.DEBUG &&
3203            Performance.MEASURE &&
3204            mon_LogSyncStatistics)
3205                {
3206                        Monitor.logMessage("number of times someone waited = " +
3207                                                   mon_numLogFlushWaits +
3208                                                   "\nnumber of times flush is called = " +
3209                                                   mon_flushCalls +
3210                                                   "\nnumber of sync is called = " +
3211                                                   mon_syncCalls +
3212                                                   "\ntotal number of bytes written to log = " +
3213                                                   LogAccessFile.mon_numBytesToLog +
3214                                                   "\ntotal number of writes to log file = " +
3215                                                   LogAccessFile.mon_numWritesToLog);
3216                }
3217                
3218 
3219                // delete obsolete log files,left around by earlier crashes
3220                if(corrupt == null && ! logArchived() && !keepAllLogs && !ReadOnlyDB)        
3221                        deleteObsoleteLogfiles();
3222 
3223        if( logDevice != null)
3224            logStorageFactory.shutdown();
3225        logStorageFactory = null;
3226        }
3227 
3228 
3229 
3230        /* delete the log files, that might have been left around if we crashed
3231         * immediately after the checkpoint before truncations of logs completed.
3232         * see bug no: 3519 , for more details.
3233         */
3234 
3235        private void deleteObsoleteLogfiles(){
3236                StorageFile logDir;
3237                //find the first  log file number that is  useful
3238                long firstLogNeeded = getFirstLogNeeded(currentCheckpoint);
3239        if (firstLogNeeded == -1)
3240                        return;
3241 
3242        // when  backup is in progress, log files that are yet to
3243        // be copied to the backup should not be deleted,  even 
3244        // if they are not required  for crash recovery.
3245        if(backupInProgress) {
3246            long logFileNeededForBackup = logFileToBackup;
3247            // check if the log file number is yet to be copied 
3248            // to the backup is less than the log file required 
3249            // for crash recovery, if it is then make the first 
3250            // log file that should not be deleted is the log file 
3251            // that is yet to  be copied to the backup.  
3252            if (logFileNeededForBackup < firstLogNeeded)
3253                firstLogNeeded = logFileNeededForBackup;
3254        }
3255 
3256                try{
3257                        logDir = getLogDirectory();
3258                }catch (StandardException se)
3259                {
3260                        if (SanityManager.DEBUG)
3261                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
3262                                        SanityManager.DEBUG(DBG_FLAG, "error opening log segment dir");
3263                        return;
3264                }
3265                        
3266                String[] logfiles = privList(logDir);
3267                if (logfiles != null)
3268                {
3269                        StorageFile uselessLogFile = null;
3270                        long fileNumber;
3271                        for(int i=0 ; i < logfiles.length; i++)
3272                        {
3273                                // delete the log files that are not needed any more
3274                                if(logfiles[i].startsWith("log") && logfiles[i].endsWith(".dat"))
3275                                {
3276                                        fileNumber = Long.parseLong(logfiles[i].substring(3, (logfiles[i].length() -4)));
3277                                        if(fileNumber < firstLogNeeded )
3278                                        {
3279                                                uselessLogFile = logStorageFactory.newStorageFile(logDir, logfiles[i]);
3280                                                if (privDelete(uselessLogFile))
3281                                                {
3282                                                        if (SanityManager.DEBUG)
3283                                                        {
3284                                                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
3285                                                                        SanityManager.DEBUG(DBG_FLAG, "truncating obsolete log file " + uselessLogFile.getPath());
3286                                                        }
3287                                                }
3288                                                else
3289                                                {
3290                                                        if (SanityManager.DEBUG)
3291                                                        {
3292                                                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
3293                                                                        SanityManager.DEBUG(DBG_FLAG, "Fail to truncate obsolete log file " + uselessLogFile.getPath());
3294                                                        }
3295                                                }
3296                                        }
3297                                }
3298                        }
3299                }
3300        }
3301 
3302        /*
3303         * Serviceable methods
3304         */
3305 
3306        public boolean serviceASAP()
3307        {
3308                return false;
3309        }
3310 
3311        // @return true, if this work needs to be done on a user thread immediately
3312        public boolean serviceImmediately()
3313        {
3314                return false;
3315        }        
3316 
3317 
3318        public void getLogFactoryProperties(PersistentSet set)
3319                 throws StandardException
3320        {
3321                String lsInterval;
3322                String cpInterval;
3323                if(set == null)
3324                {
3325                        lsInterval=PropertyUtil.getSystemProperty(org.apache.derby.iapi.reference.Property.LOG_SWITCH_INTERVAL);
3326                        cpInterval=PropertyUtil.getSystemProperty(org.apache.derby.iapi.reference.Property.CHECKPOINT_INTERVAL);
3327                }else
3328                {
3329                        lsInterval = PropertyUtil.getServiceProperty(set, org.apache.derby.iapi.reference.Property.LOG_SWITCH_INTERVAL);
3330                        cpInterval = PropertyUtil.getServiceProperty(set, org.apache.derby.iapi.reference.Property.CHECKPOINT_INTERVAL);
3331                }
3332 
3333                /* log switch interval */
3334                if (lsInterval != null)
3335                {
3336                        logSwitchInterval = Integer.parseInt(lsInterval);
3337                                        // make sure checkpoint and log switch interval are within range
3338                        if (logSwitchInterval < LOG_SWITCH_INTERVAL_MIN)
3339                                logSwitchInterval = LOG_SWITCH_INTERVAL_MIN;
3340                        else if (logSwitchInterval > LOG_SWITCH_INTERVAL_MAX)
3341                                logSwitchInterval = LOG_SWITCH_INTERVAL_MAX;
3342                }
3343 
3344                /* checkpoint interval */
3345                if (cpInterval != null)
3346                {
3347                        checkpointInterval = Integer.parseInt(cpInterval);
3348                        if (checkpointInterval < CHECKPOINT_INTERVAL_MIN)
3349                                checkpointInterval = CHECKPOINT_INTERVAL_MIN;
3350                        else if(checkpointInterval  > CHECKPOINT_INTERVAL_MAX)
3351                                checkpointInterval = CHECKPOINT_INTERVAL_MAX;
3352                }
3353        }
3354 
3355        public int performWork(ContextManager context)
3356        {
3357                synchronized(this)
3358                {
3359                        if (corrupt != null)
3360                                return Serviceable.DONE; // don't do this again.
3361                }
3362 
3363                // check to see if checkpointInterval and logSwitchInterval has changed
3364                AccessFactory af = 
3365            (AccessFactory)Monitor.getServiceModule(this, AccessFactory.MODULE);
3366 
3367                try
3368                {
3369                        if (af != null)
3370                        {
3371                                TransactionController tc = null;
3372                                try
3373                                {
3374                                        tc = af.getAndNameTransaction(
3375                            context, AccessFactoryGlobals.SYS_TRANS_NAME);
3376 
3377                                        getLogFactoryProperties(tc);
3378                                }
3379                                finally
3380                                {
3381                                        if (tc != null)
3382                                                tc.commit();
3383                                }
3384                        }
3385 
3386                        // checkpoint will start its own internal transaction on the current
3387                        // context.
3388                        rawStoreFactory.checkpoint();
3389                }
3390                catch (StandardException se)
3391                {
3392                        Monitor.logTextMessage(MessageId.LOG_CHECKPOINT_EXCEPTION);
3393                        logErrMsg(se);
3394                }
3395        catch (ShutdownException shutdown)
3396        {
3397            // If we are shutting down, just ignore the error and let the 
3398            // system go down without printing errors to the log.
3399        }
3400 
3401                checkpointDaemonCalled = false;
3402 
3403                return Serviceable.DONE;
3404        }
3405 
3406 
3407        /*
3408        ** Implementation specific methods
3409        */
3410 
3411        /**
3412                Append length bytes of data to the log prepended by a long log instant
3413                and followed by 4 bytes of length information.
3414 
3415                <P>
3416                This method is synchronized to ensure log records are added sequentially
3417                to the end of the log.
3418 
3419                <P>MT- single threaded through this log factory.  Log records are
3420                appended one at a time.
3421 
3422                @exception StandardException Log Full.
3423 
3424        */
3425        protected long appendLogRecord(byte[] data, int offset, int length,
3426                        byte[] optionalData, int optionalDataOffset, int optionalDataLength) 
3427                 throws StandardException
3428        {
3429                long instant;
3430                boolean testIncompleteLogWrite = false;
3431 
3432                if (ReadOnlyDB)
3433        {
3434                        throw StandardException.newException(
3435                SQLState.LOG_READ_ONLY_DB_UPDATE);
3436        }
3437 
3438                if (length <= 0)
3439        {
3440            throw StandardException.newException(
3441                    SQLState.LOG_ZERO_LENGTH_LOG_RECORD);
3442        }
3443 
3444                // RESOLVE: calculate checksum here
3445                if (SanityManager.DEBUG)
3446                {
3447                        if (SanityManager.DEBUG_ON(TEST_LOG_INCOMPLETE_LOG_WRITE))
3448                        {
3449                                /// /// /// /// /// /// /// /// /// /// 
3450                                //
3451                                // go into this alternate route instead
3452                                //
3453                                /// /// /// /// /// /// /// /// /// /// 
3454                                return logtest_appendPartialLogRecord(data, offset, length,
3455                                                                                                          optionalData,
3456                                                                                                          optionalDataOffset,
3457                                                                                                          optionalDataLength);
3458 
3459                        }
3460 
3461                }
3462 
3463                try
3464                {
3465                        if (SanityManager.DEBUG)
3466                        {
3467                                if (SanityManager.DEBUG_ON(TEST_LOG_FULL))
3468                                        testLogFull();        // if log is 'full' this routine will throw an
3469                                                                // exception 
3470                        }
3471 
3472                        synchronized (this)
3473                        {
3474                                // has someone else found a problem in the raw store?
3475                                if (corrupt != null)
3476                {
3477                                        throw StandardException.newException(
3478                            SQLState.LOG_STORE_CORRUPT, corrupt);
3479                }
3480 
3481                                if (logOut == null)
3482                {
3483                                        throw StandardException.newException(SQLState.LOG_NULL);
3484                }
3485 
3486                                /*
3487                                 * NOTE!!
3488                                 *
3489                                 * subclass which logs special record to the stream depends on
3490                                 * the EXACT byte sequence of the following segment of code.  
3491                                 * If you change this, not only will you need to write upgrade
3492                                 * code for this class, you also need to find all the subclass
3493                                 * which write log record to log stream directly to make sure 
3494                                 * they are OK
3495                                 */
3496 
3497                                // see if the log file is too big, if it is, switch to the next
3498                                // log file
3499                                if ((endPosition + LOG_RECORD_OVERHEAD + length) >=
3500                                        LogCounter.MAX_LOGFILE_SIZE)
3501                                {
3502                                        switchLogFile();
3503 
3504                                        // still too big??  Giant log record?
3505                                        if ((endPosition + LOG_RECORD_OVERHEAD + length) >=
3506                                                LogCounter.MAX_LOGFILE_SIZE) 
3507                    {
3508                                                throw StandardException.newException(
3509                                SQLState.LOG_EXCEED_MAX_LOG_FILE_SIZE, 
3510                                new Long(logFileNumber), 
3511                                new Long(endPosition), 
3512                                new Long(length), 
3513                                new Long(LogCounter.MAX_LOGFILE_SIZE));
3514                    }
3515                                }
3516 
3517                                //reserve the space for the checksum log record
3518                                endPosition += logOut.reserveSpaceForChecksum(length, logFileNumber,endPosition);
3519 
3520                                // don't call currentInstant since we are already in a
3521                                // synchronzied block 
3522                                instant = 
3523                    LogCounter.makeLogInstantAsLong(logFileNumber, endPosition);
3524 
3525                logOut.writeLogRecord(
3526                    length, instant, data, offset, 
3527                    optionalData, optionalDataOffset, optionalDataLength);
3528 
3529                                if (optionalDataLength != 0) 
3530                {
3531                                        if (SanityManager.DEBUG)
3532                                        {
3533                                                if (optionalData == null)
3534                                                        SanityManager.THROWASSERT(
3535                                                        "optionalDataLength = " + optionalDataLength +
3536                                                        " with null Optional data");
3537 
3538                                                if (optionalData.length <
3539                                                                                         (optionalDataOffset+optionalDataLength))
3540                                                        SanityManager.THROWASSERT(
3541                                                        "optionalDataLength = " + optionalDataLength +
3542                                                        " optionalDataOffset = " + optionalDataOffset + 
3543                                                        " optionalData.length = " + optionalData.length);
3544                                        }
3545                                }
3546 
3547                                endPosition += (length + LOG_RECORD_OVERHEAD);
3548                        }
3549                }
3550                catch (IOException ioe)
3551                {
3552                        throw markCorrupt(StandardException.newException(
3553                    SQLState.LOG_FULL, ioe));
3554                }
3555 
3556                return instant;
3557        }
3558 
3559        /*
3560         * Misc private functions to access the log
3561         */
3562 
3563        /**
3564                Get the current log instant - this is the log instant of the Next log
3565                record to be written out
3566                <P> MT - This method is synchronized to ensure that it always points to
3567                the end of a log record, not the middle of one. 
3568        */
3569        protected synchronized long currentInstant()
3570        {
3571                return LogCounter.makeLogInstantAsLong(logFileNumber, endPosition);
3572        }
3573 
3574        protected synchronized long endPosition()
3575        {
3576                return endPosition;
3577        }
3578 
3579        /**
3580                Return the current log file number.
3581 
3582                <P> MT - this method is synchronized so that
3583                it is not in the middle of being changed by swithLogFile
3584        */
3585        private synchronized long getLogFileNumber()
3586        {
3587                return logFileNumber;
3588        }
3589 
3590        /** 
3591                Get the first valid log instant - this is the beginning of the first
3592                log file
3593 
3594                <P>MT- synchronized on this
3595        */
3596        private synchronized long firstLogInstant()
3597        {
3598                return LogCounter.makeLogInstantAsLong(firstLogFileNumber, LOG_FILE_HEADER_SIZE);
3599        }
3600 
3601        /**
3602                Flush the log such that the log record written with the instant 
3603        wherePosition is guaranteed to be on disk.
3604 
3605                <P>MT - only one flush is allowed to be taking place at any given time 
3606                (RESOLVE: right now it single thread thru the log factory while the log
3607                is frozen) 
3608 
3609                @exception StandardException cannot sync log file
3610 
3611        */
3612        protected void flush(long fileNumber, long wherePosition) throws StandardException
3613        {
3614 
3615                long potentialLastFlush = 0;
3616 
3617                synchronized (this)
3618                {
3619                        if (Performance.MEASURE)
3620                                mon_flushCalls++;
3621                        try
3622                        {
3623                                boolean waited;
3624                                do
3625                                {
3626                                        // THIS CORRUPT CHECK MUST BE FIRST, before any check that
3627                                        // sees if the log has already been flushed to this
3628                                        // point. This is based upon the assumption that every
3629                                        // dirty page in the cache must call flush() before it is
3630                                        // written out.  has someone else found a problem in the
3631                                        // raw store?
3632 
3633                                        if (corrupt != null)
3634                                        {
3635                                                throw StandardException.newException(
3636                                SQLState.LOG_STORE_CORRUPT, corrupt);
3637                                        }
3638 
3639                                        // now check if database is frozen
3640                                        while (isFrozen)
3641                                        {
3642                                                try 
3643                                                {
3644                                                        wait();
3645                                                } 
3646                                                catch (InterruptedException ie) 
3647                                                {
3648                                                        throw StandardException.interrupt(ie);
3649                                                }
3650                                        }
3651 
3652                                        // if we are just testing to see to see the database is 
3653                    // frozen or corrupt (wherePosition == INVALID_LOG_INSTANT)
3654                    // then we can return now.
3655                                        // if the log file is already flushed up to where we are 
3656                                        // interested in, just return.
3657                                        if (wherePosition == LogCounter.INVALID_LOG_INSTANT ||
3658                                                fileNumber < logFileNumber ||
3659                                                wherePosition < lastFlush)
3660                                        {
3661                                                return;
3662                                        }
3663 
3664                                        // if we are not corrupt and we are in the middle of redo, 
3665                    // we know the log record has already been flushed since we haven't written any log 
3666                    // yet.
3667                                        if (recoveryNeeded && inRedo) 
3668                                        {
3669                                                return;
3670                                        }
3671 
3672 
3673                                        if (SanityManager.DEBUG)
3674                                        {
3675                                                if (fileNumber > getLogFileNumber())
3676                                                        SanityManager.THROWASSERT(
3677                                                          "trying to flush a file that is not there yet " +
3678                                                                 fileNumber + " " + logFileNumber);
3679 
3680                                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
3681                        {
3682                                                        SanityManager.DEBUG(
3683                                DBG_FLAG, "Flush log to " + wherePosition);
3684                        }
3685                                        }
3686 
3687                                        // There could be multiple threads who wants to flush the 
3688                    // log file, see if I can be the one.
3689                                        if (logBeingFlushed)
3690                                        {
3691                                                waited = true;
3692                                                try
3693                                                {
3694                                                        if (Performance.MEASURE)
3695                                                                mon_numLogFlushWaits++;
3696                                                        wait();        // release log semaphore to let non-flushing
3697                                                        // threads log stuff while all the flushing 
3698                                                        // threads wait.
3699 
3700                                                        // now we continue back to see if the sync
3701                                                        // we waited for, flushed the portion
3702                                                        // of log we are interested in.
3703                                                }
3704                                                catch (InterruptedException ie)
3705                                                {
3706                                                        throw StandardException.interrupt(ie);
3707                                                }
3708                                        }
3709                                        else
3710                                        {
3711                                                waited = false;
3712 
3713                                                // logBeingFlushed is false, I am flushing the log now.
3714                                                if(!isWriteSynced)
3715                                                {
3716                                                        // Flush any data from the buffered log
3717                                                        logOut.flushLogAccessFile();
3718                                                }else
3719                                                {
3720                                                        //add active buffers to dirty buffer list
3721                                                        //to flush to the disk.
3722                                                        logOut.switchLogBuffer();
3723                                                }
3724 
3725                                                potentialLastFlush = endPosition; // we will flush to to the end
3726 
3727                                                // once logBeingFlushed is set, need to release
3728                                                // the logBeingFlushed flag in finally block.
3729                                                logBeingFlushed = true;        
3730                                        }
3731 
3732                                } while (waited) ;
3733                                // if I have waited, go down do loop again - hopefully,
3734                                // someone else have already flushed it for me already.
3735                        }
3736                        catch (IOException ioe)
3737                        {
3738                                throw markCorrupt(StandardException.newException(
3739                    SQLState.LOG_CANNOT_FLUSH, 
3740                    ioe,
3741                    getLogFileName(logFileNumber).getPath()));
3742                        }
3743                } // unfreeze log manager to accept more log records
3744 
3745                boolean syncSuceed = false;
3746                try
3747                {
3748                        if (SanityManager.DEBUG)
3749                        {
3750                                SanityManager.ASSERT(logBeingFlushed, 
3751                                                                         "flushing log without logBeingFlushed set");
3752                                SanityManager.ASSERT(potentialLastFlush > 0,
3753                                                                         "potentialLastFlush not set");
3754 
3755                                if (SanityManager.DEBUG_ON(TEST_LOG_FULL))
3756                                        testLogFull();
3757 
3758                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
3759                                        SanityManager.DEBUG(DBG_FLAG, "Begin log sync...");
3760                        }
3761 
3762                        
3763                        if (Performance.MEASURE)
3764                                mon_syncCalls++;
3765 
3766                        if (isWriteSynced)
3767                        {
3768                                //LogAccessFile.flushDirtyBuffers() will allow only one write
3769                                //sync at a time, flush requests will get queued 
3770                                logOut.flushDirtyBuffers();
3771                        }
3772                        else
3773                        {
3774                                if (!logNotSynced)
3775                                    logOut.syncLogAccessFile();
3776                        }
3777 
3778                        syncSuceed = true;
3779 
3780                        if (SanityManager.DEBUG)
3781                        {
3782                                if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
3783                                        SanityManager.DEBUG(DBG_FLAG, "end log sync.");
3784                        }
3785                }
3786                catch (SyncFailedException sfe) 
3787                {
3788                        throw markCorrupt(StandardException.newException(
3789                SQLState.LOG_CANNOT_FLUSH, 
3790                sfe,
3791                getLogFileName(logFileNumber).getPath()));
3792                }
3793                catch (IOException ioe)
3794                {
3795                        throw markCorrupt(StandardException.newException(
3796                SQLState.LOG_CANNOT_FLUSH, 
3797                ioe,
3798                getLogFileName(logFileNumber).getPath()));
3799                }
3800                finally
3801                {
3802                        synchronized(this)
3803                        {
3804                                logBeingFlushed = false; // done flushing
3805 
3806                                // update lastFlush under synchronized this instead of synchronized(logOut)
3807                                if (syncSuceed)
3808                                {
3809                                        lastFlush = potentialLastFlush;
3810                                }
3811 
3812 
3813                                // We may actually have flushed more than that because someone
3814                                // may have done a logOut.flushBuffer right before the sync
3815                                // call. But this is guarenteed to be flushed.
3816                                notifyAll();
3817                        }
3818                }
3819 
3820                
3821                // get checkpoint Daemon to work
3822                if ((logWrittenFromLastCheckPoint + potentialLastFlush) > checkpointInterval &&
3823                                        checkpointDaemon != null &&        !checkpointDaemonCalled && !inLogSwitch)
3824                {
3825                        // following synchronized block is required to make 
3826                        // sure only one checkpoint request get scheduled.
3827                        synchronized(this)
3828                        {
3829                                // recheck if checkpoint is still required, it is possible some other
3830                                // thread might have already scheduled a checkpoint and completed it. 
3831                                if ((logWrittenFromLastCheckPoint + potentialLastFlush) > checkpointInterval &&
3832                                        checkpointDaemon != null &&        !checkpointDaemonCalled && !inLogSwitch)
3833                                {
3834                                        checkpointDaemonCalled = true;
3835                                        checkpointDaemon.serviceNow(myClientNumber);
3836                                }
3837                        }
3838 
3839                }else
3840                {
3841                        // switch the log if required, this case will occur 
3842                        // if log switch interval is less than the checkpoint interval
3843                        // other wise , checkpoint daemon would be doing log switches along
3844                        // with the checkpoints.
3845                        if (potentialLastFlush > logSwitchInterval &&
3846                                !checkpointDaemonCalled && !inLogSwitch)
3847                        {
3848                                // following synchronized block is required to make sure only
3849                                // one thread switches the log file at a time.
3850                                synchronized(this)
3851                                {
3852                                        // recheck if log switch is still required, it is possible some other
3853                                        // thread might have already switched the log file. 
3854                                        if (potentialLastFlush > logSwitchInterval &&
3855                                                !checkpointDaemonCalled && !inLogSwitch)
3856                                        {
3857                                                inLogSwitch = true;
3858                                                switchLogFile();
3859                                        }
3860                                }
3861                        }
3862                }
3863        }
3864 
3865    /**
3866     * Utility routine to call sync() on the input file descriptor.
3867     * <p> 
3868    */
3869    private void syncFile( StorageRandomAccessFile raf) 
3870        throws StandardException
3871    {
3872        for( int i=0; ; )
3873        {
3874            // 3311: JVM sync call sometimes fails under high load against NFS 
3875            // mounted disk.  We re-try to do this 20 times.
3876            try
3877            {
3878                raf.sync( false);
3879 
3880                // the sync succeed, so return
3881                break;
3882            }
3883            catch (IOException ioe)
3884            {
3885                i++;
3886                try
3887                {
3888                    // wait for .2 of a second, hopefully I/O is done by now
3889                    // we wait a max of 4 seconds before we give up
3890                    Thread.sleep(200);
3891                }
3892                catch( InterruptedException ie )
3893                {   
3894                    //does not matter weather I get interrupted or not
3895                }
3896 
3897                if( i > 20 )
3898                {
3899                    throw StandardException.newException(
3900                                SQLState.LOG_FULL, ioe);
3901                }
3902            }
3903        }
3904    }
3905 
3906 
3907        /**
3908          Open a forward scan of the transaction log.
3909 
3910          <P> MT- read only
3911          @exception StandardException  Standard cloudscape exception policy
3912        */
3913        public LogScan openForwardsFlushedScan(LogInstant startAt)
3914                 throws StandardException
3915        {
3916                checkCorrupt();
3917 
3918                // no need to flush the buffer as it's a flushed scan
3919 
3920                return new FlushedScan(this,((LogCounter)startAt).getValueAsLong());
3921        }
3922 
3923 
3924        /**
3925          Get a forwards scan
3926 
3927          @exception StandardException Standard Cloudscape error policy
3928          */
3929        public LogScan openForwardsScan(LogInstant startAt,LogInstant stopAt)
3930                 throws StandardException
3931        {
3932                try
3933                {
3934                        long startLong;
3935                
3936                        if (startAt == null)
3937                                startLong = LogCounter.INVALID_LOG_INSTANT;
3938                        else
3939                                startLong = ((LogCounter)startAt).getValueAsLong();
3940 
3941                        return openForwardsScan(startLong, stopAt);
3942                }
3943 
3944                catch (IOException ioe) 
3945                {
3946                        throw markCorrupt(StandardException.newException(
3947                                        SQLState.LOG_IO_ERROR, ioe));
3948                }
3949 
3950        }
3951 
3952        public final boolean databaseEncrypted()
3953        {
3954                return databaseEncrypted;
3955        }
3956 
3957 
3958    /*
3959     * Set that the database is encrypted, all the data in the 
3960     * transaction log  should be encrypted. 
3961     */
3962    public  void setDatabaseEncrypted() 
3963    {
3964        databaseEncrypted = true;
3965    }
3966 
3967    /*
3968     * setup log for encryption. 
3969     */
3970    public  void setupLogEncryption() throws StandardException
3971    {
3972        // switch the database to a new log file, so that 
3973        // new encrytion will start on new log file. 
3974        switchLogFile();
3975    }
3976 
3977 
3978        /**
3979                @see RawStoreFactory#encrypt
3980                @exception StandardException Standard Cloudscape Error Policy
3981         */
3982        public int encrypt(byte[] cleartext, int offset, int length, 
3983                                                  byte[] ciphertext, int outputOffset)
3984                 throws StandardException
3985        {
3986        return rawStoreFactory.encrypt(cleartext, offset, length, 
3987                                       ciphertext, outputOffset, false);
3988        }
3989 
3990        /**
3991                @see RawStoreFactory#decrypt
3992                @exception StandardException Standard Cloudscape Error Policy
3993         */
3994        public int decrypt(byte[] ciphertext, int offset, int length,
3995                                                         byte[] cleartext, int outputOffset)
3996                 throws StandardException
3997        {
3998                return rawStoreFactory.decrypt(ciphertext, offset, length, cleartext, outputOffset);
3999        }
4000 
4001        /**
4002                return the encryption block size used during encrypted db creation
4003         */
4004        public int getEncryptionBlockSize()
4005        {
4006                return rawStoreFactory.getEncryptionBlockSize();
4007        }
4008 
4009        /**
4010           returns the length that will make the data to be multiple of encryption
4011           block size based on the given length. Block cipher algorithms like DES 
4012           and Blowfish ..etc  require their input to be an exact multiple of the block size.
4013        */
4014        public int getEncryptedDataLength(int length)
4015        {
4016                if ((length % getEncryptionBlockSize()) != 0)
4017                {
4018                        return length + getEncryptionBlockSize() - (length % getEncryptionBlockSize());
4019                }
4020 
4021                return length;
4022        }
4023 
4024 
4025 
4026        /**
4027          Get the instant of the first record which was not
4028          flushed.
4029 
4030          <P>This only works after running recovery the first time.
4031          <P>MT - RESOLVE:
4032          */
4033    public synchronized LogInstant getFirstUnflushedInstant()
4034        {
4035                if (SanityManager.DEBUG)
4036                        SanityManager.ASSERT(logFileNumber > 0 && lastFlush > 0);
4037 
4038                return new LogCounter(logFileNumber,lastFlush);
4039        }
4040 
4041 
4042        /**
4043         * Backup restore - stop sending log record to the log stream
4044         * @exception StandardException Standard Cloudscape error policy
4045         */
4046        public void freezePersistentStore() throws StandardException
4047        {
4048                // if I get into this synchronized block, I know I am not in the middle
4049                // of a write because writing to the log file is synchronized under this.
4050                synchronized(this)
4051                {
4052                        isFrozen = true;
4053                }                        
4054        }
4055 
4056        /**
4057         * Backup restore - start sending log record to the log stream
4058         * @exception StandardException Standard Cloudscape error policy
4059         */
4060        public void unfreezePersistentStore() throws StandardException
4061        {
4062                synchronized(this)
4063                {
4064                        isFrozen = false;
4065                        notifyAll();
4066                }                        
4067        }
4068 
4069        /**
4070         * Backup restore - is the log being archived to some directory?
4071         * if log archive mode is enabled return true else false
4072         */
4073        public boolean logArchived()
4074        {
4075                return logArchived;
4076        }
4077 
4078        /**
4079           Check to see if a database has been upgraded to the required
4080           level in order to use a store feature.
4081           @param requiredMajorVersion  required database Engine major version
4082           @param requiredMinorVersion  required database Engine minor version
4083           @return True if the database has been upgraded to the required level, false otherwise.
4084        **/
4085        boolean checkVersion(int requiredMajorVersion, int requiredMinorVersion) 
4086        {
4087                if(onDiskMajorVersion > requiredMajorVersion )
4088                {
4089                        return true;
4090                }
4091                else
4092                {
4093                        if(onDiskMajorVersion == requiredMajorVersion &&  
4094                           onDiskMinorVersion >= requiredMinorVersion)
4095                                return true;
4096                }
4097                
4098                return false;
4099        }
4100 
4101        /*
4102        ** Sending information to the user without throwing exception.
4103        ** There are times when unusual external or system related things happen in
4104        ** the log which the user may be interested in but which doesn't impede on
4105        ** the normal operation of the store.  When such an event occur, just send
4106        ** a message or a warning message to the user rather than throw an
4107        ** exception, which will rollback a user transaction or crash the database.
4108        **
4109        ** logErrMsg - sends a warning message to the user 
4110        */
4111 
4112 
4113        /**
4114                Print error message to user about the log
4115                MT - not needed, informational only
4116        */
4117        protected void logErrMsg(String msg)
4118        {
4119               logErrMsgForDurabilityTestModeNoSync();
4120                Monitor.logTextMessage(MessageId.LOG_BEGIN_ERROR);
4121                Monitor.logMessage(msg);
4122                Monitor.logTextMessage(MessageId.LOG_END_ERROR);
4123        }
4124 
4125        /**
4126                Print error message to user about the log
4127                MT - not needed, informational only
4128        */
4129        protected void logErrMsg(Throwable t)
4130        {
4131                logErrMsgForDurabilityTestModeNoSync();
4132                if (corrupt != null)
4133                {
4134                        Monitor.logTextMessage(MessageId.LOG_BEGIN_CORRUPT_STACK);
4135                        printErrorStack(corrupt);
4136                        Monitor.logTextMessage(MessageId.LOG_END_CORRUPT_STACK);
4137                }
4138 
4139                if (t != corrupt)
4140                {
4141                        Monitor.logTextMessage(MessageId.LOG_BEGIN_ERROR_STACK);
4142                        printErrorStack(t);
4143                        Monitor.logTextMessage(MessageId.LOG_END_ERROR_STACK);
4144                }
4145        }
4146 
4147 
4148    /**
4149     * In case of boot errors, and if database is either booted
4150     * with derby.system.durability=test or was previously at any time booted in
4151     * this mode, mention in the error message that the error is probably 
4152     * because the derby.system.durability was set. 
4153     * Dont want to waste time to resolve issues in such
4154     * cases
4155     * <p>
4156     * MT - not needed, informational only
4157     */
4158    private void logErrMsgForDurabilityTestModeNoSync()
4159    {
4160        if (logNotSynced || wasDBInDurabilityTestModeNoSync)
4161        {
4162            Monitor.logTextMessage(
4163                MessageId.LOG_DURABILITY_TESTMODE_NO_SYNC_ERR,
4164                Property.DURABILITY_PROPERTY,
4165                Property.DURABILITY_TESTMODE_NO_SYNC);
4166        }
4167    }
4168 
4169    /**
4170     * print stack trace from the Throwable including
4171     * its nested exceptions 
4172     * @param t trace starts from this error
4173     */
4174        private void printErrorStack(Throwable t)
4175        {
4176                ErrorStringBuilder esb = 
4177            new ErrorStringBuilder(Monitor.getStream().getHeader());
4178                esb.stackTrace(t);
4179        Monitor.logMessage(esb.get().toString());
4180        esb.reset();
4181    }
4182 
4183 
4184        /**
4185         *  Testing support
4186         */
4187        /** 
4188                Writes out a partial log record - takes the appendLogRecord.
4189                Need to shutdown the database before another log record gets written,
4190                or the database is not recoverable.
4191        */
4192        private long logtest_appendPartialLogRecord(byte[] data, int offset, 
4193                                                                                                int        length,
4194                                                                                                byte[] optionalData, 
4195                                                                                                int optionalDataOffset, 
4196                                                                                                int optionalDataLength)
4197                throws StandardException
4198        {
4199                if (SanityManager.DEBUG)
4200                {
4201                        int bytesToWrite = 1;
4202 
4203                        String TestPartialLogWrite = PropertyUtil.getSystemProperty(TEST_LOG_PARTIAL_LOG_WRITE_NUM_BYTES);
4204                        if (TestPartialLogWrite != null)
4205                        {
4206                                bytesToWrite = Integer.valueOf(TestPartialLogWrite).intValue();
4207                        }
4208 
4209                        Monitor.logMessage("TEST_LOG_INCOMPLETE_LOG_WRITE: writing " + bytesToWrite + 
4210                                   " bytes out of " + length + " + " + LOG_RECORD_OVERHEAD + " log record");
4211 
4212                        long instant;
4213                        try
4214                        {
4215                                                                
4216                                synchronized (this)
4217                                {
4218                                        // reserve the space for the checksum log record
4219                                        // NOTE:  bytesToWrite include the log record overhead.
4220                                        endPosition += 
4221                                                logOut.reserveSpaceForChecksum(((length + LOG_RECORD_OVERHEAD) 
4222                                                                                                                < bytesToWrite ? length :
4223                                                                                                                (bytesToWrite - LOG_RECORD_OVERHEAD)),
4224                                                                                                           logFileNumber,endPosition);
4225                                        instant = currentInstant();
4226 
4227                                        //check if the length of the records to be written is 
4228                                        //actually smaller than the number of bytesToWrite 
4229                                        if(length + LOG_RECORD_OVERHEAD < bytesToWrite)
4230                                                endPosition += (length + LOG_RECORD_OVERHEAD);
4231                                        else
4232                                                endPosition += bytesToWrite;
4233 
4234                                        while(true)                // so we can break out without returning out of
4235                                                // sync block...
4236                                        {
4237                                                        if (bytesToWrite < 4)
4238                                                        {
4239                                                                int shift = 3;
4240                                                                while(bytesToWrite-- > 0)
4241                                                                {
4242                                                                        logOut.write((byte)((length >>> 8*shift) & 0xFF));
4243                                                                        shift--;
4244                                                                }
4245                                                                break;
4246                                                        }
4247 
4248                                                        // the length before the log record
4249                                                        logOut.writeInt(length);
4250                                                        bytesToWrite -= 4;
4251 
4252                                                        if (bytesToWrite < 8)
4253                                                        {
4254                                                                int shift = 7;
4255                                                                while(bytesToWrite-- > 0)
4256                                                                {
4257                                                                        logOut.write((byte)((instant >>> 8*shift) & 0xFF));
4258                                                                        shift--;
4259                                                                }
4260                                                                break;
4261                                                        }
4262 
4263                                                        // the log instant
4264                                                        logOut.writeLong(instant);
4265                                                        bytesToWrite -= 8;
4266 
4267                                                        if (bytesToWrite < length)
4268                                                        {
4269                                                                int dataLength = length - optionalDataLength;
4270                                                                if(bytesToWrite < dataLength)
4271                                                                        logOut.write(data, offset,bytesToWrite);
4272                                                                else
4273                                                                {
4274                                                                        logOut.write(data, offset, dataLength);
4275                                                                        bytesToWrite -= dataLength ;
4276                                                                        if(optionalDataLength != 0 && bytesToWrite > 0)
4277                                                                                logOut.write(optionalData, optionalDataOffset, bytesToWrite);
4278                                                                }
4279                                                                break;
4280                                                        }
4281 
4282                                                        // the log data
4283                                                        logOut.write(data, offset, length - optionalDataLength);
4284                                                        //write optional data
4285                                                        if(optionalDataLength != 0)
4286                                                                logOut.write(optionalData, optionalDataOffset, optionalDataLength);
4287 
4288                                                        bytesToWrite -= length;
4289 
4290                                                        if (bytesToWrite < 4)
4291                                                        {
4292                                                                int shift = 3;
4293                                                                while(bytesToWrite-- > 0)
4294                                                                {
4295                                                                        logOut.write((byte)((length >>> 8*shift) & 0xFF));
4296                                                                        shift--;
4297                                                                }
4298                                                                break;
4299                                                        }
4300 
4301                                                        // the length after the log record
4302                                                        logOut.writeInt(length);
4303                                                        break;
4304 
4305                                                }
4306 
4307                                                // do make sure the partial write gets on disk by sync'ing it
4308                                                flush(logFileNumber, endPosition);
4309 
4310                                        }
4311 
4312 
4313                        }
4314                        catch (IOException ioe)
4315                        {
4316                                throw StandardException.newException(SQLState.LOG_FULL, ioe);
4317                        }
4318 
4319                        return instant;
4320                }
4321                return 0;
4322        }
4323 
4324        /**
4325                Simulate a log full condition
4326 
4327                if TEST_LOG_FULL is set to true, then the property
4328                TEST_RECORD_TO_FILL_LOG indicates the number of times this function is
4329                call before an IOException simulating a log full condition is raised.
4330 
4331                If TEST_RECORD_TO_FILL_LOG is not set, it defaults to 100 log record
4332        */
4333        protected void testLogFull() throws IOException
4334        {
4335                if (SanityManager.DEBUG)
4336                {
4337                        if (test_numRecordToFillLog < 0)
4338                        {
4339                                String RecordToFillLog = PropertyUtil.getSystemProperty(TEST_RECORD_TO_FILL_LOG);
4340                                if (RecordToFillLog != null)
4341                                        test_numRecordToFillLog = Integer.valueOf(RecordToFillLog).intValue();
4342                                else
4343                                        test_numRecordToFillLog = 100;
4344                        }
4345 
4346                        if (++test_logWritten > test_numRecordToFillLog)
4347                                throw new IOException("TestLogFull " + test_numRecordToFillLog +
4348                                                                          " written " + test_logWritten);
4349 
4350                }        
4351        }
4352 
4353        /**
4354         * Get the log file to Simulate a log corruption 
4355         * FOR UNIT TESTING USAGE ONLY 
4356        */
4357        public StorageRandomAccessFile getLogFileToSimulateCorruption(long filenum) throws IOException, StandardException
4358        {
4359                if (SanityManager.DEBUG)
4360                {
4361                        //long filenum = LogCounter.getLogFileNumber(logInstant);
4362                        //                        long filepos = LogCounter.getLogFilePosition(logInstant);
4363                        StorageFile fileName = getLogFileName(filenum);
4364                        StorageRandomAccessFile log = null;
4365                        return privRandomAccessFile(fileName, "rw");
4366                }
4367                
4368                return null;
4369 
4370        }
4371        
4372 
4373        /*********************************************************************
4374         * Log Testing
4375         * 
4376         * Implementations may use these strings to simulate error conditions for
4377         * testing purposes.
4378         *
4379         *********************************************************************/
4380 
4381        /**
4382          Set to true if we want the checkpoint to only switch the log but not
4383          actually do the checkpoint
4384        */
4385        public static final String TEST_LOG_SWITCH_LOG = SanityManager.DEBUG ? "TEST_LOG_SWITCH_LOG" : null ;
4386 
4387        /**
4388          Set to true if we want the up comming log record to be only partially
4389          written.  The database is corrupted if not immediately shutdown.
4390          Set TEST_LOG_PARTIAL_LOG_WRITE_NUM_BYTES to the number of bytes to write
4391          out, default is 1 byte.
4392        */
4393        public static final String TEST_LOG_INCOMPLETE_LOG_WRITE = SanityManager.DEBUG ? "TEST_LOG_INCOMPLETE_LOG_WRITE" : null;
4394 
4395        /**
4396          Set to the number of bytes we want the next log record to actually write
4397          out, only used when TEST_LOG_INCOMPLETE_LOG_WRITE is on.  Default is 1
4398          byte.
4399        */
4400        public static final String TEST_LOG_PARTIAL_LOG_WRITE_NUM_BYTES = SanityManager.DEBUG ? "derbyTesting.unittest.partialLogWrite" : null;
4401 
4402        /**
4403          Set to true if we want to simulate a log full condition
4404        */
4405        public static final String TEST_LOG_FULL = 
4406        SanityManager.DEBUG ? "TEST_LOG_FULL" : null;
4407 
4408        /**
4409          Set to true if we want to simulate a log full condition while switching log
4410        */
4411        public static final String TEST_SWITCH_LOG_FAIL1 = 
4412        SanityManager.DEBUG ? "TEST_SWITCH_LOG_FAIL1" : null;
4413        public static final String TEST_SWITCH_LOG_FAIL2 = 
4414        SanityManager.DEBUG ? "TEST_SWITCH_LOG_FAIL2" : null;
4415 
4416 
4417        /**
4418          Set to the number of log record we want to write before the log is
4419          simulated to be full.
4420        */
4421        public static final String TEST_RECORD_TO_FILL_LOG = 
4422        SanityManager.DEBUG ? "derbyTesting.unittest.recordToFillLog" : null;
4423 
4424        /**
4425         * Set to true if we want to simulate max possible log file number is 
4426     * being used.
4427        */
4428        public static final String TEST_MAX_LOGFILE_NUMBER = 
4429        SanityManager.DEBUG ? "testMaxLogFileNumber" : null;
4430 
4431        
4432        //enable the log archive mode
4433        public void enableLogArchiveMode() throws StandardException
4434        {
4435 
4436                //if the log archive mode is already enabled; thre is nothing to do
4437                if(!logArchived)
4438                {
4439                        logArchived = true;
4440                        AccessFactory af = 
4441            (AccessFactory)Monitor.getServiceModule(this, AccessFactory.MODULE);
4442 
4443                        if (af != null)
4444                        {
4445                                TransactionController tc = null;
4446                                tc = af.getTransaction(
4447                        ContextService.getFactory().getCurrentContextManager());
4448                                tc.setProperty(Property.LOG_ARCHIVE_MODE , "true", true);
4449                        }
4450                }
4451        }
4452 
4453        // disable the log archive mode
4454        public void disableLogArchiveMode() throws StandardException
4455        {
4456                AccessFactory af = 
4457            (AccessFactory)Monitor.getServiceModule(this, AccessFactory.MODULE);
4458                if (af != null)
4459                {
4460                        TransactionController tc = null;
4461                        tc = af.getTransaction(ContextService.getFactory().getCurrentContextManager());
4462                        tc.setProperty(Property.LOG_ARCHIVE_MODE , "false", true);
4463                }
4464        logArchived = false;
4465        }
4466 
4467        //delete the online archived log files
4468        public void deleteOnlineArchivedLogFiles()
4469        {
4470                deleteObsoleteLogfiles();
4471        }
4472 
4473 
4474        /*
4475         * start the transaction log backup, transaction log is  is required
4476         * to bring the database to the consistent state on restore. 
4477 
4478         * All the log files that are created after the backup starts 
4479         * should be kept around until they are copied into the backup,
4480         * even if there are checkpoints when backup is in progress. 
4481         *
4482         * copy the log control files to the backup (the checkpoint recorded in the
4483     * control files is the backup checkpoint), Restore will use the checkpoint 
4484     * info in these control files to perform recovery to bring 
4485         * the database to the consistent state.  and find first log file 
4486         * that need to be copied into the backup to bring the database
4487         * to the consistent state on restore. 
4488         * 
4489     * In the end, existing log files that are needed to recover from the backup 
4490     * checkpoint are copied into the backup, any log that gets generated after
4491     * this call are copied into the backup after all the all the information 
4492     * in the data containers is  written to the backup, when endLogBackup() 
4493     * is called.
4494         *
4495     * @param toDir - location where the log files should be copied to.
4496     * @exception StandardException Standard Derby error policy
4497         *
4498         */
4499        public void startLogBackup(File toDir) throws StandardException
4500        {
4501                
4502                // synchronization is necessary to make sure NO parallel 
4503                // checkpoint happens when the current checkpoint information 
4504                // is being copied to the backup.
4505 
4506                synchronized(this) 
4507                {
4508                        // wait until the thread that is doing the checkpoint completes it.
4509                        while(inCheckpoint)
4510                        {
4511                                try
4512                                {
4513                                        wait();
4514                                }        
4515                                catch (InterruptedException ie)
4516                                {
4517                                        throw StandardException.interrupt(ie);
4518                                }        
4519                        }
4520                
4521                        backupInProgress = true;
4522                
4523                        // copy the control files. 
4524                        StorageFile fromFile;
4525                        File toFile;
4526                        // copy the log control file
4527                        fromFile = getControlFileName();
4528                        toFile = new File(toDir,fromFile.getName());
4529                        if(!privCopyFile(fromFile, toFile))
4530                        {
4531                                throw StandardException.newException(SQLState.RAWSTORE_ERROR_COPYING_FILE,
4532                                                                                                         fromFile, toFile);
4533                        }
4534 
4535                        // copy the log mirror control file
4536                        fromFile = getMirrorControlFileName();
4537                        toFile = new File(toDir,fromFile.getName());
4538                        if(!privCopyFile(fromFile, toFile))
4539                        {
4540                                throw StandardException.newException(SQLState.RAWSTORE_ERROR_COPYING_FILE,
4541                                                                                                         fromFile, toFile);
4542                        }
4543 
4544                        // find the first  log file number that is  active
4545                        logFileToBackup = getFirstLogNeeded(currentCheckpoint);
4546                }
4547 
4548                // copy all the log files that has to go into the backup 
4549                backupLogFiles(toDir, getLogFileNumber()-1);
4550        }        
4551 
4552        /*
4553         * copy the log files into the given backup location
4554     * @param toDir - location where the log files should be copied to.
4555     * @param lastLogFileToBackup - last log file that needs to be copied.
4556         **/
4557        private void backupLogFiles(File toDir, long lastLogFileToBackup) 
4558        throws StandardException
4559        {
4560 
4561                while(logFileToBackup <= lastLogFileToBackup)
4562                {
4563                        StorageFile fromFile = getLogFileName(logFileToBackup);
4564                        File toFile = new File(toDir, fromFile.getName());
4565                        if(!privCopyFile(fromFile, toFile))
4566                        {
4567                                throw StandardException.newException(SQLState.RAWSTORE_ERROR_COPYING_FILE,
4568                                                                                                         fromFile, toFile);
4569                        }
4570                        logFileToBackup++;
4571                }
4572        }
4573 
4574        /*
4575         * copy all the log files that has to go into  the backup
4576         * and mark that backup is compeleted. 
4577     *
4578     * @param toDir - location where the log files should be copied to.
4579     * @exception StandardException Standard Derby error policy
4580         */
4581        public void endLogBackup(File toDir) throws StandardException
4582        {
4583                long lastLogFileToBackup;
4584                if (logArchived)
4585                {
4586                        // when the log is being  archived for roll-frward recovery
4587                        // we would like to switch to  a new log file.
4588                        // otherwise during restore  logfile in the backup could 
4589                        // overwrite the more uptodate log files in the 
4590                        // online log path. And also we would like to mark the end
4591                        // marker for the log file other wise during roll-forward recovery,
4592                        // if we see a log file with fuzzy end , we think that is the 
4593                        // end of the recovery.
4594                        switchLogFile();
4595                        lastLogFileToBackup = getLogFileNumber()-1 ;
4596                }else
4597                {
4598                        // for a plain online backup partiall filled up log file is ok, 
4599                        // no need to do a log switch.
4600                        lastLogFileToBackup = getLogFileNumber();        
4601                }
4602 
4603                // backup all the log that got generated after the backup started.
4604                backupLogFiles(toDir, lastLogFileToBackup);
4605 
4606                // mark that backup is completed.
4607                backupInProgress = false;
4608        }
4609 
4610 
4611        /*
4612         * backup is not in progress any more, it failed for some reason.
4613         **/
4614        public void abortLogBackup()
4615        {
4616                backupInProgress = false;
4617        }
4618 
4619 
4620        // Is the transaction in rollforward recovery
4621        public boolean inRFR()
4622        {
4623                /*
4624                 *Logging System does not differentiate between the
4625                 *crash-recovery and a rollforward recovery.
4626                 *Except in case of rollforward atttempt on 
4627                 *read only databases to check for pending Transaction.
4628                 *(See the comments in recovery() function)
4629                 */
4630 
4631                if(recoveryNeeded)
4632                {
4633                        boolean readOnly = false;
4634                        try{
4635                                readOnly = !privCanWrite(getControlFileName());
4636                        }catch(StandardException se)
4637                        {
4638                                //Exception should never have come here
4639                                //because getControlFileName() is called 
4640                                //earlier at boot time, if there were problems
4641                                //it should have showed up earlier.                                                        
4642                                //We just ignore this error and hope that
4643                                //datafactory must have market it as read only if that is the case.
4644                        }
4645 
4646                        readOnly = readOnly || (dataFactory == null ? false :dataFactory.isReadOnly());
4647                        return !readOnly;
4648                }else{
4649                        return false;
4650                }
4651        }
4652 
4653        /**        
4654         *        redo a checkpoint during rollforward recovery
4655        */
4656        public void checkpointInRFR(LogInstant cinstant, long redoLWM, DataFactory df) throws StandardException
4657        {
4658                //sync the data
4659                df.checkpoint();
4660 
4661                //write the log control file; this will make sure that restart of the 
4662                //rollfoward recovery will start this log instant next time instead of
4663                //from the beginning.
4664                try{
4665                        if (!writeControlFile(getControlFileName(), ((LogCounter)cinstant).getValueAsLong()))
4666                        {
4667                                throw StandardException.newException(
4668                                                                                                 SQLState.LOG_CONTROL_FILE, getControlFileName());
4669                        }
4670                }
4671                catch (IOException ioe)
4672                {
4673                        throw markCorrupt(
4674                    StandardException.newException(SQLState.LOG_IO_ERROR, ioe));
4675                }
4676                //remove the stub files
4677                df.removeDroppedContainerFileStubs(new LogCounter(redoLWM));
4678                
4679        }
4680 
4681 
4682        /**
4683         *
4684         * This function restores logs based on the  following attributes
4685         * are specified on connection URL:
4686         * Attribute.CREATE_FROM (Create database from backup if it does not exist)
4687         * Attribute.RESTORE_FROM (Delete the whole database if it exists and then 
4688     * restore it from backup)
4689         * Attribute.ROLL_FORWARD_RECOVERY_FROM:(Perform Rollforward Recovery;
4690         * except for the log directory everthing else is replced  by the copy  from
4691         * backup. log files in the backup are copied to the existing online log 
4692     * directory.
4693         *
4694         * In cases of RESTORE_FROM whole databases directoy is 
4695         * is removed in Directory.java while restoring service.properties
4696         * so even the log directory is removed.
4697         * In case of CREATE_FROM , log directoy will not exist if 
4698         * we came so far bacause it should fail if a database already exists.
4699         * In case ROLL_FORWARD_RECOVERY_FROM log directotry should not be removed.
4700         * So only thing that needs to be done here is create a
4701         * a log directory if it does not exists and copy the 
4702         * log files(including control files) that exists in the backup from which 
4703     * we are are trying to restore the database to the onlie log directory.
4704         */
4705        private boolean restoreLogs(Properties properties) throws StandardException
4706        {
4707 
4708                String backupPath =null;
4709                boolean isCreateFrom = false; 
4710                boolean isRestoreFrom = false;
4711 
4712                //check if the user requested for restore/recovery/create from backup
4713                backupPath = properties.getProperty(Attribute.CREATE_FROM);
4714        if (backupPath != null) {
4715            isCreateFrom = true;
4716        } else {
4717                        backupPath = properties.getProperty(Attribute.RESTORE_FROM);
4718            if (backupPath != null) {
4719                isRestoreFrom = true;
4720            } else {
4721                backupPath = properties.getProperty(
4722                                  Attribute.ROLL_FORWARD_RECOVERY_FROM);
4723                // if the backup is not NULL then it is a rollforward recovery.
4724            }
4725        }
4726 
4727                if(backupPath !=null)
4728                {
4729                        if(!isCreateFrom){
4730                                if(logDevice == null){
4731                                        /**
4732                                         * In restoreFrom/rollForwardRecoveryFrom mode when no 
4733                     * logDevice on URL then the log is restored to the same 
4734                     * location where the log was when backup was taken.
4735                                         * In createFrom mode behaviour is same as when create=true,
4736                                         * i.e unless user specifies the logDevice on URL, log will
4737                     * be copied to the database home dir.
4738                                         * Note: LOG_DEVICE_AT_BACKUP will get set if log is not in
4739                                         * default location(db home). 
4740                                         */
4741                                        logDevice = 
4742                        properties.getProperty(Property.LOG_DEVICE_AT_BACKUP);
4743                                }
4744                        }         
4745        
4746            getLogStorageFactory();
4747                        StorageFile logDir;
4748                        logDir = logStorageFactory.newStorageFile( 
4749                             LogFactory.LOG_DIRECTORY_NAME);
4750                                
4751                        //remove the log directory in case of restoreFrom 
4752                        //if it exist, this happens if the log device is on seperate
4753                        //location than the db home.
4754                        if (isRestoreFrom && logDevice != null)
4755                        {
4756                                if(!privRemoveDirectory(logDir))
4757                                {
4758                                        //it may be just a file, try deleting it
4759                                        if(!privDelete(logDir))
4760                    {
4761                                                throw StandardException.newException(
4762                            SQLState.UNABLE_TO_REMOVE_DATA_DIRECTORY,
4763                            getLogDirPath( logDir));
4764                    }
4765                                }
4766                        }
4767 
4768            // if it is a create/restore from backup, 
4769            // create the log directory.
4770            if (isCreateFrom || isRestoreFrom) {
4771                createLogDirectory();
4772            }
4773 
4774                        File backupLogDir = new File(backupPath, LogFactory.LOG_DIRECTORY_NAME);
4775                        String[] logfilelist = privList(backupLogDir);
4776                        if(logfilelist !=null)
4777                        {
4778                                for (int i = 0; i < logfilelist.length; i++) 
4779                                {
4780                                        File blogFile = new File(backupLogDir, logfilelist[i]);
4781                                        StorageFile clogFile = logStorageFactory.newStorageFile(logDir, logfilelist[i]);
4782                                        if(!privCopyFile(blogFile , clogFile))
4783                                        {
4784                                                throw
4785                                                        StandardException.newException(SQLState.UNABLE_TO_COPY_LOG_FILE, blogFile, clogFile);
4786                                        }
4787                                }
4788                        }else
4789                        {
4790                                throw StandardException.newException(SQLState.LOG_DIRECTORY_NOT_FOUND_IN_BACKUP,backupLogDir);
4791                        }
4792                        //we need to switch the log file after redo while
4793                        //doing recovery from backups, otherwise we will 
4794                        //be replacing updated log after a restore withe 
4795                        // a log in the backup on next restore.
4796                        logSwitchRequired = true;
4797 
4798            // log is restored from backup.
4799            return true;
4800                } else {
4801            // log is not restored from backup.
4802            return false;
4803        }
4804        }
4805 
4806        /*preallocate the given log File to the logSwitchInterval size;
4807         *file is extended by writing zeros after the header till 
4808         *the log file size the set by the user.
4809         */        
4810        private void preAllocateNewLogFile(StorageRandomAccessFile log) throws IOException, StandardException
4811    {
4812        //preallocate a file by writing zeros into it . 
4813 
4814        if (SanityManager.DEBUG)
4815        {
4816            int currentPostion = (int)log.getFilePointer();
4817            SanityManager.ASSERT(currentPostion == LOG_FILE_HEADER_SIZE, 
4818                                 "New Log File Is not Correctly Initialized");
4819        }
4820 
4821        int amountToWrite = logSwitchInterval - LOG_FILE_HEADER_SIZE ;
4822        int bufferSize = logBufferSize * 2;
4823        byte[] emptyBuffer = new byte[bufferSize];
4824        int nWrites = amountToWrite/bufferSize;
4825        int remainingBytes = amountToWrite % bufferSize;
4826        
4827        try{
4828            while(nWrites-- > 0)
4829                log.write(emptyBuffer);
4830 
4831            if(remainingBytes !=0)
4832                log.write(emptyBuffer , 0 ,remainingBytes);
4833 
4834            //sync the file
4835            syncFile(log);
4836        }catch(IOException ie)
4837        {
4838            //ignore io exceptions during preallocations
4839            //because this more for performance improvements
4840            //system shoulf work fine even without preallocations. 
4841 
4842            //RESOLVE: If  the exception is because of no 
4843            //space, might be good idea to trigger a checkpoint.
4844 
4845            //In debug mode throw the exception 
4846            if (SanityManager.DEBUG)
4847            {
4848                throw ie;
4849            }
4850        }
4851    } // end of preAllocateNewLogFile
4852 
4853 
4854        /**
4855         * open the given log file name for writes; if file can not be 
4856         * be opened in write sync mode then disable the write sync mode and 
4857         * open the file in "rw" mode.
4858          */
4859        private StorageRandomAccessFile openLogFileInWriteMode(StorageFile logFile) throws IOException
4860        {
4861                StorageRandomAccessFile log;
4862                try{
4863                        log = privRandomAccessFile(logFile, "rws");
4864                }catch(FileNotFoundException ex)
4865                {
4866                        // Normally this exception should never occur. For some reason
4867                        // currently on Mac JVM 1.4.2 FileNotFoundException exception is
4868                        // thrown if a file is opened in "rws" mode and if it already
4869                        // exists. Please refere to Derby-1 for more/ details on this issue.
4870                        // Temporary workaround to avoid this problem is to make the logging 
4871                        // system use file sync mechanism. 
4872 
4873                        // disable the write sync and open the file in "rw" mode. 
4874                        isWriteSynced = false;
4875                        log = privRandomAccessFile(logFile, "rw");
4876                }
4877                
4878                return log ;
4879        }
4880 
4881 
4882    private String getLogDirPath( StorageFile logDir)
4883    {
4884        if( logDevice == null)
4885            return logDir.toString();
4886        return logDevice + logStorageFactory.getSeparator() + logDir.toString();
4887    } // end of getLogDirPath
4888 
4889    /*
4890        Following  methods require Priv Blocks to run under a security manager.
4891    */
4892        private int action;
4893        private StorageFile activeFile;
4894        private File toFile;
4895        private String activePerms;
4896 
4897    protected boolean privExists(StorageFile file)
4898    {
4899                return runBooleanAction(0, file);
4900        }
4901 
4902    protected boolean privDelete(StorageFile file)
4903    {
4904                return runBooleanAction(1, file);
4905    }
4906 
4907    private synchronized StorageRandomAccessFile privRandomAccessFile(StorageFile file, String perms)
4908        throws IOException
4909    {
4910                action = 2;
4911        activeFile = file;
4912        activePerms = perms;
4913        try
4914        {
4915            return (StorageRandomAccessFile) java.security.AccessController.doPrivileged(this);
4916        }
4917        catch (java.security.PrivilegedActionException pae)
4918        {
4919            throw (IOException) pae.getException();
4920        }
4921    }
4922 
4923    protected boolean privCanWrite(StorageFile file)
4924    {
4925                return runBooleanAction(3, file);
4926    }
4927 
4928    protected boolean privMkdirs(StorageFile file)
4929    {
4930                return runBooleanAction(4, file);
4931    }
4932 
4933        private synchronized String[] privList(File file)
4934    {
4935                action = 8;
4936        toFile = file;
4937 
4938        try
4939        {
4940                        return (String[]) java.security.AccessController.doPrivileged(this);
4941                }
4942        catch (java.security.PrivilegedActionException pae)
4943        {
4944            return null;
4945        }
4946        }
4947    
4948        private synchronized String[] privList(StorageFile file)
4949    {
4950                action = 5;
4951        activeFile = file;
4952 
4953        try
4954        {
4955                        return (String[]) java.security.AccessController.doPrivileged(this);
4956                }
4957        catch (java.security.PrivilegedActionException pae)
4958        {
4959            return null;
4960        }
4961        }
4962 
4963 
4964        private synchronized boolean privCopyFile(StorageFile from, File to)
4965        {
4966                action = 6;
4967                activeFile = from;
4968                toFile = to;
4969        try
4970        {
4971                        return ((Boolean) java.security.AccessController.doPrivileged(this)).booleanValue();
4972                }
4973        catch (java.security.PrivilegedActionException pae)
4974        {
4975            return false;
4976        }        
4977        }
4978 
4979        private synchronized boolean privCopyFile(File from, StorageFile to)
4980        {
4981                action = 9;
4982                activeFile = to;
4983                toFile = from;
4984        try
4985        {
4986                        return ((Boolean) java.security.AccessController.doPrivileged(this)).booleanValue();
4987                }
4988        catch (java.security.PrivilegedActionException pae)
4989        {
4990            return false;
4991        }        
4992        }
4993 
4994        private boolean privRemoveDirectory(StorageFile file)
4995        {
4996                return runBooleanAction(7, file);
4997        }
4998 
4999 
5000        private synchronized boolean runBooleanAction(int action, StorageFile file) {
5001                this.action = action;
5002                this.activeFile = file;
5003 
5004                try {
5005                        return ((Boolean) java.security.AccessController.doPrivileged(this)).booleanValue();
5006                } catch (java.security.PrivilegedActionException pae) {
5007                        return false;
5008                }
5009        }
5010 
5011 
5012        
5013 
5014        public final Object run() throws IOException {
5015                switch (action) {
5016                case 0:
5017                        // SECURITY PERMISSION - MP1
5018                        return ReuseFactory.getBoolean(activeFile.exists());
5019                case 1:
5020                        // SECURITY PERMISSION - OP5
5021           return ReuseFactory.getBoolean(activeFile.delete());
5022                case 2:
5023                        // SECURITY PERMISSION - MP1 and/or OP4
5024                        // dependening on the value of activePerms
5025                        return activeFile.getRandomAccessFile(activePerms);
5026                case 3:
5027                        // SECURITY PERMISSION - OP4
5028                        return ReuseFactory.getBoolean(activeFile.canWrite());
5029                case 4:
5030                        // SECURITY PERMISSION - OP4
5031                        return ReuseFactory.getBoolean(activeFile.mkdirs());
5032                case 5:
5033                        // SECURITY PERMISSION - MP1
5034                        return activeFile.list();
5035                case 6:
5036                        // SECURITY PERMISSION - OP4 (Have to check these codes ??)
5037                        return ReuseFactory.getBoolean(FileUtil.copyFile(logStorageFactory, activeFile, toFile));
5038                case 7:
5039                        // SECURITY PERMISSION - OP4
5040            if( ! activeFile.exists())
5041                return ReuseFactory.getBoolean( true);
5042                        return ReuseFactory.getBoolean(activeFile.deleteAll());
5043        case 8:
5044            return toFile.list();
5045        case 9:
5046            return ReuseFactory.getBoolean(FileUtil.copyFile( logStorageFactory, toFile, activeFile));
5047 
5048                default:
5049                        return null;
5050                }
5051        }
5052}

[all classes][org.apache.derby.impl.store.raw.log]
EMMA 2.0.5312 (C) Vladimir Roubtsov