Xtext is an extended EBNF language for describing domain-specific languages (DSLs). The extended scope of expressions (compared to EBNF) allows the definition of a corresponding meta-model. Source code can be read by the Xtext parser component MweReader. MweReader is a configurable MWE workflow component, able to transform any artefact in conformity to an Xtext DSL into the target model.

However, some use cases may require transformation of more than one source file. In such cases, every single artefact needs its own workflow setup. Even then it is not possible

• defining multiple input files and directories (recursively read all files in a directory and its subdirectories ending with the DSL’s file extension)
• merging all target models into one

In such cases, another workflow component may help, called MweBulkReader. It combines all features of MweReader plus the aforementioned ones: Define multiple resources, files or folders. Folders are searched for all files with the DSL’s file extension (or a custom one). For each and every artefact MweReader is called, with a consecutively numbered output slot name (“model_{1}” set per default). In the end all created output slots are merged into one slot (named “model” or redefined by property “outputSlot”).

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Vector;
import java.util.Map.Entry;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.eclipse.emf.common.util.URI;
import org.eclipse.emf.ecore.resource.Resource;
import org.eclipse.emf.mwe.core.WorkflowContext;
import org.eclipse.emf.mwe.core.issues.Issues;
import org.eclipse.emf.mwe.core.lib.AbstractWorkflowComponent2;
import org.eclipse.emf.mwe.core.monitor.ProgressMonitor;
import org.eclipse.xtext.ISetup;
import org.eclipse.xtext.MweReader;
import org.eclipse.xtext.resource.XtextResourceFactory;

/**
 * Identical to MweReader but is further able to read a list of resources
 * specified at <sourceDirectory>s and <Uri>s.
 *
 * If a resource is a directory it is recursively scanned for files, filtered by extension.
 * File extension is defined by registered Xtext DSL or can be manually changed with <extension>.
 * Files are parsed into enumerated model slots 'model_{0}'s or <outputSlotFormat>s
 * The parser class has to be set by <register>.
 * The parser's class path may be set with <classpathURIContext>.
 * The parser's validation flag can be set using <validate>.
 * A list of all parsed models is made available at slot 'model' or <outputSlot>
 *
 * @author Andreas Rentschler
 */
public class MweBulkReader extends AbstractWorkflowComponent2 {

	public static final String DEFAULT_OUTPUT_SLOT_FORMAT = "model_{0}";
	public static final String DEFAULT_OUTPUT_SLOT = "model";

	private final Log log = LogFactory.getLog(getClass());

	private ISetup setup = null;

	public void setRegister(ISetup setup) {
		this.setup  = setup;
	}

	private String extension = null;

	public void setExtension(String extension) {
		this.extension = extension;
	}

	private String outputSlotFormat = DEFAULT_OUTPUT_SLOT_FORMAT;

	public void setOutputSlotFormat(String outputSlotFormat) {
		this.outputSlotFormat = outputSlotFormat;
	}

	private String outputSlot = DEFAULT_OUTPUT_SLOT;

	/**
	 * Sets an accumulation model slot name. If set (non-null and non-empty),
	 * then a {@link List} of all models read in, is stored in the designated
	 * model slot. This way, using the {@link #getModelSlots()} method is not
	 * necessary.
	 */
	public void setOutputSlot(String outputSlot) {
		this.outputSlot = outputSlot;
	}

	private List<String> sourceDirectories = new ArrayList<String>();

	public void addSourceDirectory(String path) {
		sourceDirectories.add(path);
	}

	public void addUri(String path) {
		sourceDirectories.add(path);
	}

	private Object classpathURIContext;

	public Object getClasspathURIContext() {
        return classpathURIContext;
    }

    public void setClasspathURIContext(Object classpathURIContext) {
        this.classpathURIContext = classpathURIContext;
    }

	private boolean validate = true;

	public void setValidate(boolean validate) {
		this.validate = validate;
	}

	@Override
	protected void checkConfigurationInternal(Issues issues) {
		if (setup == null)
			issues.addError(this,"No setup has been registered (property 'register')");
		if (extension == null) {
			// register file extension
			setup.createInjectorAndDoEMFRegistration();

			// take latest file extension registered to Xtext
			for (Entry<String, Object> e : Resource.Factory.Registry.INSTANCE.getExtensionToFactoryMap().entrySet()) {
				if (e.getValue() instanceof XtextResourceFactory) {
					extension = e.getKey();
				}
			}
			if (extension != null)
				log.info(getComponentName() + "(" + getId() + "): No explicit file extension configured, found '" + extension + "'");
		}
		if (extension == null)
			issues.addError(this,"No file extension has been registered (property 'register')");
		if (sourceDirectories.isEmpty())
			issues.addError(this,"No resource uri configured (property 'uri')");
		if (outputSlot == null)
			issues.addWarning("no explicit output slot configured, using + '" + outputSlot + "'");
	}

	private int freeSlot = 0;
	private List<String> modelSlots = new ArrayList<String>();
	private WorkflowContext ctx;
	private ProgressMonitor monitor;
	private Issues issues;

	@Override
	protected void invokeInternal(WorkflowContext ctx, ProgressMonitor monitor, Issues issues) {
		this.ctx = ctx;
		this.monitor = monitor;
		this.issues = issues;

		for (String directory : sourceDirectories) {
			File resource = new File(directory);

			if (!resource.exists()) {
				issues.addError("'" + resource + "' does not exist!");
			} else {
				if (resource.isDirectory())
					addComponentsForDirectory(resource);
				else
					addComponentForFile(resource);
			}
		}

		// merge all models into one
		List<Object> list = new ArrayList<Object>();
		for (String slotName : modelSlots) {
			if (ctx.get(slotName) == null) {
				// ensure that list of models doesn't contain EVoid elements
				log.info(getComponentName() + "(" + getId() + "): Skipping empty slot '" + slotName + "'");
				continue;
			}
			list.add(ctx.get(slotName));
		}
		ctx.set(outputSlot, list);
	}

	@Override
	public String getLogMessage() {
		return "Loading " + sourceDirectories.size() + (sourceDirectories.size() == 1? " resource" : " resources");
	}

	private void addComponentsForDirectory(File dir) {
		log.info(getComponentName() + "(" + getId() + "): Registering models from path '" + dir + "'");
		Collection<File> files = listFiles(dir, new FilenameFilter() {
			public boolean accept(File directory, String filename) {
				if (extension == null) return false;

				int index = filename.lastIndexOf('.');
				if (index < 0) {
					return false;
				}
				return filename.substring(index + 1).equals(extension);
			}
		}, true);
		for (File f : files) {
			addComponentForFile(f);
		}
	}

	private void addComponentForFile(File f) {
		String uri = null;
		try {
			uri = URI.createFileURI(f.getCanonicalPath()).toFileString();
		} catch (IOException e) {
			e.printStackTrace();
		}
		String slot = MessageFormat.format(outputSlotFormat, freeSlot++);
		modelSlots.add(slot);

		String shortUri = (uri.length() > 57) ? "..." + uri.substring(uri.length() - 60) : uri;
		log.info(getComponentName() + "(" + getId() + "): Parsing file:" + shortUri + " into slot '" + slot + "'");

		MweReader reader = new MweReader();
		reader.setContainer(getContainer());
		reader.setLocation(getLocation());
		//reader.setSkipOnErrors(true);
		reader.setClasspathURIContext(classpathURIContext);
		reader.setValidate(validate);
		reader.setRegister(setup);
		reader.setOutputSlot(slot);
		reader.setUri("file:" + uri);
		try {
			reader.invoke(ctx, monitor, issues);
		} catch (IndexOutOfBoundsException e) {
			// catch weird bug (files producing no elements (not even a container element, maybe some comments)
			log.info(getComponentName() + "(" + getId() + "): Skipping file because it is empty");
		} catch (Exception e) {
			log.info(getComponentName() + "(" + getId() + "): Skipping file because of exception " + e);
		}
	}

	// list files in a directory filtered, optionally recursively
	private Collection<File> listFiles(File directory,
			FilenameFilter filter, boolean recurse) {
		Vector<File> files = new Vector<File>();
		File[] entries = directory.listFiles();

		for (File entry : entries) {
			if (filter == null || filter.accept(directory, entry.getName())) {
				files.add(entry);
			}

			if (recurse && entry.isDirectory()) {
				files.addAll(listFiles(entry, filter, recurse));
			}
		}
		return files;
	}

}

This is one example of application in a workflow configuration .mwe:

<workflow>
...
	<!-- parse all Xtext artefacts in files ${sourceFile1}, ${sourceFile2} and folders ${sourcePath1}, ${sourcePath2} into slot 'mergedModel' -->
	<component id="dsl2model" class="my.classpath.to.bulk.reader.MweBulkReader">
		<!-- this class has been generated by the xtext generator -->
		<register class="my.classpath.to.dsl.strategy.XXXStandaloneSetup"/>
		<uri value="file:${sourceFile1}" />
		<uri value="file:${sourceFile2}" />
		<sourceDirectory value="${sourcePath1}" />
		<sourceDirectory value="${sourcePath2}" />
		<outputSlot value="mergedModel" />
	</component>
...
</workflow>