/*
 * Copyright
 *   2005 axYus - www.axyus.com
 *   2005 C.Marchand - christophe.marchand@axyus.com
 *
 * This file is part of XEMELIOS.
 *
 * XEMELIOS is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * XEMELIOS is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with XEMELIOS; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

package fr.gouv.finances.cp.xemelios.importers;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.Vector;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;

import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import fr.gouv.finances.cp.utils.Pair;
import fr.gouv.finances.cp.utils.StringUtilities;
import fr.gouv.finances.cp.xemelios.common.Constants;
import fr.gouv.finances.cp.xemelios.common.DematTransform;
import fr.gouv.finances.cp.xemelios.common.FileInfo;
import fr.gouv.finances.cp.xemelios.common.ToolException;
import fr.gouv.finances.cp.xemelios.common.config.DocumentModel;
import fr.gouv.finances.cp.xemelios.common.config.EtatModel;
import fr.gouv.finances.cp.xemelios.data.DataAccessException;
import fr.gouv.finances.cp.xemelios.data.DataConfigurationException;
import fr.gouv.finances.cp.xemelios.data.DataLayerManager;
import fr.gouv.finances.cp.xemelios.data.impl.sqlconfig.TDocument;
import fr.gouv.finances.cp.xemelios.importers.transformers.EtatTransformeur;
import fr.gouv.finances.cp.xemelios.utils.FileUtils;

/**
 * This class imports a document in the underlying data layer.
 * First it splits the document in small files that contains only one
 * importable element.
 * Then, if the "Etat" configuration defines an Xsl to apply before importing,
 * it applies it.
 * Finally, it delegates import to the data layer.
 *
 * This class has been fully-rewritten since Xemelios 3.0, replacing the old
 * <tt>DefaultImporteur</tt> one.
 *
 * @since 3.0
 *
 * @author chm
 */
public class DefaultImporter extends EtatImporteur {
    private static Logger logger = Logger.getLogger(DefaultImporter.class);
    
    //private Hashtable<DocumentModel,Pair> budgets=new Hashtable<DocumentModel,Pair>();
    //private Hashtable<DocumentModel,Pair> collectivites=new Hashtable<DocumentModel,Pair>();
    
//    private File documentsConfigDir = null;
    
    public DefaultImporter() {
        super();
//        documentsConfigDir = new File(System.getProperty(Constants.SYS_PROP_DOC_DEF_DIR));
    }
    
    @Override
    public void setDocument(DocumentModel dm) {
        this.dm = dm;
    }
    
    
    /**
     * Importe le fichier specifie.
     * @param f Le fichier a importer
     * @see fr.gouv.finances.cp.xemelios.importers.EtatImporteur#importe(java.io.File)
     */
    @Override
    protected FileInfo importFile(File f) throws Exception {
logger.debug("importing "+f.getAbsolutePath());
        getImpSvcProvider().startLongWait();
        String fileEncoding = getFileEncoding(f);
        String xmlVersion = getFileXmlVersion(f);
        File tmpDir = new File(System.getProperty("java.io.tmpdir"));
        tmpDir = new File(tmpDir,Constants.NOM_APP);
        tmpDir.mkdirs();
        logger.debug("tmpDir="+tmpDir.getPath());
        
        File fHeader = File.createTempFile("header-","",tmpDir);
        File fRef = File.createTempFile("ref-","",tmpDir);
        File fFooter = File.createTempFile("footer-","",tmpDir);
        XmlSplitter xs = splitFile(fHeader,fRef, fFooter, tmpDir,f,fileEncoding);
        getImpSvcProvider().endLongWait();
        Pair collectivite = new Pair();
        Pair codeBudget = new Pair();
        Pair referenceNomenclature = new Pair();
        
        if(col == null){
        	collectivite = xs.getCollectivite();
        	getImpSvcProvider().setCollectivite(collectivite);
        } else {
        	collectivite = col;
        }
        
        if(bdg == null){
        	codeBudget = xs.getBudget();
        	getImpSvcProvider().setBudget(codeBudget);
        } else {
        	codeBudget = bdg;
        }
        
        Vector<File> dataFiles = xs.getDataFiles();
        String header = FileUtils.readTextFile(fHeader,fileEncoding);
        String footer = FileUtils.readTextFile(fFooter,fileEncoding);
        String ref = FileUtils.readTextFile(fRef,fileEncoding);
        
        int docCount=1;
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        dbf.setNamespaceAware(true);
        DocumentBuilder db = dbf.newDocumentBuilder();
        
// load documents
        TDocument pc = DataLayerManager.getImplementation().getPersistenceConfig(dm).getLayer(DataLayerManager.getImplementation().getLayerName()).getDocument(dm.getId());
        if(collectivite==null || collectivite.key==null || collectivite.libelle==null) {
            try {
                collectivite = getCollectivite(dm,f.getName());
            } catch(ToolException tEx) {
                if(tEx.getErrorCode()==ToolException.ERROR_INTERRUPTED)
                    getImpSvcProvider().displayMessage(tEx.getMessage(),ImportServiceProvider.MSG_TYPE_INFORMATION);
                removeTempDir();
                return null;
            }
        }
        if(codeBudget==null  || codeBudget.key==null || codeBudget.libelle==null || codeBudget.libelle.equals("")) {
            try {
                codeBudget = getBudget(dm,collectivite,f.getName());
            } catch(ToolException tEx) {
                if(tEx.getErrorCode()==ToolException.ERROR_INTERRUPTED)
                    getImpSvcProvider().displayMessage(tEx.getMessage(),ImportServiceProvider.MSG_TYPE_INFORMATION);
                removeTempDir();
                return null;
            }
        }
        
        
        
        // check if document exists, and if so, if we must remove it
        if(DataLayerManager.getImplementation().isDocumentExists(dm,codeBudget,collectivite,f.getName())) {
            boolean shouldImport = false;
            try {
                shouldImport = getOverwrite(f.getName());
                if(shouldImport) {
                    // this can be very long, display the infinite waiter
                    getImpSvcProvider().startLongWait();
                    DataLayerManager.getImplementation().removeDocument(dm,codeBudget,collectivite,f.getName());
                    getImpSvcProvider().endLongWait();
                } else {
                    // TODO clean all temporary files
                    return new FileInfo();
                }
            } catch(DataAccessException daEx) {
                logger.error("DataAcessException while removing:",daEx.getCause());
                daEx.getCause().printStackTrace();
            } catch(DataConfigurationException dcEx) {
                logger.error("DataConfigurationException while removing:",dcEx.getCause());
                dcEx.getCause().printStackTrace();
            } catch(Exception ex) {
                // si exception ici, l'utilisateur veut annuler
                logger.error(ex);
                ex.printStackTrace();
                removeTempDir();
                return null;
            }
        }
        
        DataLayerManager.getImplementation().registerBudgetCollectivite(dm,codeBudget,collectivite,f.getName());
        DataLayerManager.getImplementation().saveSpecialKeys(dm,codeBudget,collectivite,f.getName(),xs.getSpecialKey1(),xs.getSpecialKey2(),xs.getSpecialKey3());
        
        for(int fileCount=0;fileCount<dataFiles.size();fileCount++) {
            if(isCancelled()) break;
            File df=dataFiles.get(fileCount);
            final int progress = (int)((fileCount+1)*100.0/dataFiles.size());
            this.getImpSvcProvider().pushCurrentProgress(progress);
            String data = FileUtils.readTextFile(df,fileEncoding);
            StringBuffer fullText = new StringBuffer();
            fullText.append("<?xml version=\"").append(xmlVersion).append("\" encoding=\"").append(fileEncoding).append("\"?>");
            fullText.append(header).append(data).append(footer);
            String sFullText = fullText.toString();
            
            Document doc = db.parse(new ByteArrayInputStream(sFullText.getBytes(fileEncoding)));
            
            // il faut retrouver de quel etat est ce document
            // on cherche si la balise root contient un dm.getEtatxxx().getBalise()
            // FIXME: be more precise in path-traversing
            EtatModel currentEtat = null;
            for(EtatModel em:dm.getEtats()) {
                String balise = em.getBalise();
                NodeList nl = doc.getElementsByTagName(balise);
                if(nl.getLength()>0) {
                    currentEtat = em;
                    break;
                } else {
                    nl = doc.getElementsByTagNameNS(dm.getNamespaces().getNamespaceURI(""),balise);
                    if(nl.getLength()>0) {
                        currentEtat = em;
                        break;
                    }
                }
            }
//logger.debug("EtatModel="+currentEtat.getId());
            // apply before-import xslt
            if(pc.getEtat(currentEtat.getId()).getImportXsltBuffer()!=null) {
            	/**
            	 * Un etat peut avoir un attribut indiquant que l'on doit appeler une classe Java afin de transformer une partie du flux.
            	 * Par exemple, la classe qui prends en compte les PJ du PES_ALLER.
            	 */
            	if (currentEtat.getTransformClass() != null) {
            		Class clazz = Class.forName(currentEtat.getTransformClass());
            		Object o = clazz.newInstance();
            		EtatTransformeur et = (EtatTransformeur)o;
            		et.transform(collectivite, fileEncoding, xmlVersion, dm.getBaseDirectory(), sFullText);
            		if (et.getReturnCode()!=EtatTransformeur.RETURN_CODE_OK) throw new Exception ("Erreur lors de la transformation du flux");
            	}
            	
                HashMap<String,Object> parameters = new HashMap<String,Object>();
                //parameters.put("encoding",fileEncoding);
                //parameters.put("xml-version",xmlVersion);
                if(bdg!=null) {
                    parameters.put("CodeBudget",bdg.key);
                    parameters.put("LibelleBudget",bdg.libelle);
                }
                if(col!=null) {
                    parameters.put("CodeCollectivite",col.key);
                    parameters.put("LibelleCollectivite",col.libelle);
                }
                InputStream is = DematTransform.transform(new ByteArrayInputStream(sFullText.getBytes(fileEncoding)),new ByteArrayInputStream(pc.getEtat(currentEtat.getId()).getImportXsltBuffer().replaceAll("##output-encoding##",fileEncoding).replaceAll("##xml-version##",xmlVersion).getBytes()),fileEncoding,parameters);

//                TransformerFactory tFactory = TransformerFactory.newInstance();
//                Transformer t = tFactory.newTransformer(new StreamSource(new File(pc.getEtat(currentEtat.getId()).getBaseDirectory(),pc.getEtat(currentEtat.getId()).getImportXsltFile())));
//                for(String key:parameters.keySet()) t.setParameter(key, parameters.get(key));
//                ByteArrayOutputStream baos = new ByteArrayOutputStream();
//                t.transform(new StreamSource(new ByteArrayInputStream(sFullText.getBytes(fileEncoding))), new StreamResult(baos));
//                InputStream is = new ByteArrayInputStream(baos.toByteArray());
                if(is==null) {
                    throw new DataConfigurationException("Erreur pendant la transformation XSL avec "+currentEtat.getParent().getBaseDirectory()+"/"+pc.getEtat(currentEtat.getId()).getImportXsltFile());
                }
                StringBuffer newText = new StringBuffer();
                int len = 0;
                byte[] buff = new byte[1024];
                do {
                    len = is.read(buff);
                    if(len>0) newText.append(new String(buff,0,len));
                } while (len > 0);
                
                sFullText = newText.toString();
                
                //logger.debug("\n***************** AFTER TRANSFORM *****************\n"+sFullText+"\n********************************************\n");
            }
            
            String docName = StringUtilities.removeFileNameSuffix(f.getName())+"-"+docCount+"."+dm.getExtension();
            if(!isCancelled()) {
                try {
                    if(!DataLayerManager.getImplementation().importElement(dm,currentEtat,codeBudget,collectivite,f.getName(),docName,sFullText,fileEncoding)) {
                        logger.warn(DataLayerManager.getImplementation().getLastWarnings());
                    }
                } catch(Throwable t) {
                    logger.error("importing element:",t);
                }
            }
            df.delete();
            docCount++;
        }
//        logger.debug("importing repository");
        if(!isCancelled()) {
            StringBuffer fullText = new StringBuffer();
            fullText.append("<?xml version=\"").append(xmlVersion).append("\" encoding=\"").append(fileEncoding).append("\" ?>");
            fullText.append(header).append(ref).append(footer);
            Document doc = db.parse(new ByteArrayInputStream(fullText.toString().getBytes(fileEncoding)));
            if(pc.getRepositoryImportXsltFile()!=null && pc.getRepositoryImportXsltFile().length()>0) {
                Document doc2 = null;
                File xslNomencl = new File(new File(dm.getBaseDirectory()),pc.getRepositoryImportXsltFile());
                String xslEncoding = FileUtils.getFileEncoding(xslNomencl);
                String xsl = FileUtils.readTextFile(xslNomencl,xslEncoding);
                xsl = xsl.replaceAll("##output-encoding##",fileEncoding);
                xsl = xsl.replaceAll("##xml-version##",xmlVersion);
                //logger.debug("XSL : \n"+xsl);
                if(xsl!=null && xsl.length()>0) {
                    HashMap<String,Object> parameters = new HashMap<String,Object>();
                    parameters.put("encoding",fileEncoding);
                    //parameters.put("xml-version",xmlVersion);
                    
                    InputStream is = null;
                    
                    is = DematTransform.transform(doc,new ByteArrayInputStream(xsl.getBytes(xslEncoding)),/*fileEncoding,*/parameters);
// DEBUG
//                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    //ByteArrayInputStream bais = new ByteArrayInputStream()
//                    byte[] buff = new byte[1024];
//                    int read = is.read(buff);
//                    while(read>0) {
//                        baos.write(buff,0,read);
//                        read = is.read(buff);
//                    }
////                    logger.debug("Apres XSL import\n"+new String(baos.toByteArray()));
//                    is = new ByteArrayInputStream(baos.toByteArray());
// END DEBUG
                    doc2 = db.parse(is);
                }
                if(doc2!=null) doc=doc2;
            }
            fullText = null;
            // String docName = "nomenclature";
            DataLayerManager.getImplementation().saveRepository(dm,collectivite,doc,xs.getReferenceNomenclature());
        } else {
            logger.debug("cancelled");
        }
        fHeader.delete();
        fRef.delete();
        fFooter.delete();
        //tmpDir.delete();
        //deltree(tmpDir);
        return xs.getFileInfo();
    }
    
    protected XmlSplitter splitFile(File fHeader, File fRef, File fFooter, File tmpDir, File fToImport, String fileEncoding) throws FileNotFoundException, UnsupportedEncodingException, SAXException, ParserConfigurationException {
        XmlSplitter xs = new XmlSplitter(
                new FileOutputStream(fHeader),
                new FileOutputStream(fRef),
                new FileOutputStream(fFooter),
                tmpDir,dm,fileEncoding);
        SAXParserFactory sf = SAXParserFactory.newInstance();
        sf.setNamespaceAware(true);
        SAXParser parser = sf.newSAXParser();
        try {
            parser.parse(fToImport,xs);
        } catch(Exception ex) {
            ex.printStackTrace();
            String position = "line "+xs.getLocator().getLineNumber()+": ";
            throw new SAXException(position+ex.getMessage(),ex);
        }
        return xs;
    }
    protected void removeTempDir() {
        File tmpDir = new File(System.getProperty("java.io.tmpdir"));
//        tmpDir = new File(tmpDir,Constants.NOM_APP);
//        if(tmpDir.exists()) {
//            File[] files = tmpDir.listFiles();
//            for(File df:files) df.delete();
//        }
        deltree(tmpDir);
    }
    protected static void deltree(File f) {
        if(f.exists()) {
            if(f.isDirectory()) {
                for(File child:f.listFiles()) {
                    if(child.isDirectory()) deltree(child);
                    else child.delete();
                }
            }
            f.delete();
        }
    }
    public boolean getOverwrite(String docName) throws Exception {
        return getImpSvcProvider().getOverwrite(docName);
    }
    
    protected Pair getBudget(DocumentModel dm, Pair collectivite, String fileName) throws ToolException {
        return getImpSvcProvider().getBudget(dm,collectivite,fileName);
    }
    protected Pair getCollectivite(DocumentModel dm, String fileName) throws ToolException {
        return getImpSvcProvider().getCollectivite(dm,fileName);
    }
    
    protected static String getFileEncoding(File f) throws IOException {
        String ret = "UTF-8";
        try {
            BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f));
            byte[] bb = new byte[255];
            int read = bis.read(bb);
            if(read<255) logger.info("while reading "+f.getName()+" for getFileEncoding(), can only read "+read+" bytes.");
            String buff = new String(bb);
            buff = buff.substring(buff.indexOf("<?"));
            buff = buff.substring(0,buff.indexOf("?>")+2);
            String header = buff;
            int encodingPos = header.indexOf("encoding=");
            if(encodingPos>=0) {
                String s = header.substring(encodingPos+9);
                if(s.startsWith("\"")) {
                    ret=s.substring(1,s.indexOf('"',1));
                } else {
                    s = s.substring(0,s.length()-2).trim();
                    if(s.indexOf(' ')>0) ret = s.substring(0,s.indexOf(' '));
                    else ret = s;
                }
            } else {
                //ret = (new InputStreamReader(new FileInputStream(f))).getEncoding();
                ret = "UTF-8";
            }
            bis.close();
        } catch(Exception ex) {}
        logger.debug("encoding="+ret);
        return ret;
    }
    protected static String getFileXmlVersion(File f) throws IOException {
        String ret = null;
        BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f));
        byte[] bb = new byte[255];
        bis.read(bb);
        String buff = new String(bb);
        buff = buff.substring(buff.indexOf("<?"));
        buff = buff.substring(0,buff.indexOf("?>")+2);
        String header = buff;
        int versionPos = header.indexOf("version=");
        if(versionPos>=0) {
            String s = header.substring(versionPos+8);
            if(s.startsWith("\"")) {
                ret=s.substring(1,s.indexOf('"',1));
            } else {
                s = s.substring(0,s.length()-2).trim();
                if(s.indexOf(' ')>0) ret = s.substring(0,s.indexOf(' '));
                else ret = s;
            }
        } else {
            ret = (new InputStreamReader(new FileInputStream(f))).getEncoding();
        }
        bis.close();
        logger.debug("version="+ret);
        return ret;
    }
    
    public static InputStream transform(Node xmlNode, InputStream xslStream, String encoding) throws IOException {
        TransformerFactory tFactory = TransformerFactory.newInstance();
        Transformer transformer;
        try {
            transformer = tFactory.newTransformer(new StreamSource(xslStream));
        } catch (TransformerConfigurationException e) {
            logger.error(e);
            return null;
        }
        ByteArrayOutputStream outArray = new ByteArrayOutputStream();
        try {
            transformer.transform(new DOMSource(xmlNode), new StreamResult(outArray));
        } catch (TransformerException e) {
            logger.error(e);
            return null;
        }
        
        return new ByteArrayInputStream(outArray.toString(encoding).getBytes());
    }
    @Override
    public String getAdditionalMessage() {
        return "";
    }
    @Override
    protected void postProcess(boolean isErrorOccured) throws Exception {
        logger.info("L'import a prit "+((System.currentTimeMillis()-start)/1000/60.0)+" mins");
    }
}
