Enable SSL for mgmt servers and agents

The port remains 8250.

The keystore saved at /etc/cloud/management/cloud.keystore. We also include one
fail-safe keystore/certificate for fallback if we are unable to generate
certificate and keystore. If we use fail-safe keystore, a warning and calltrace would be show.

Notice you need to upgrade agent, as well as systemVM's images.
This commit is contained in:
Sheng Yang 2011-04-28 16:39:21 -07:00
parent 025642801a
commit cf114fc7af
8 changed files with 484 additions and 47 deletions

View File

@ -366,28 +366,30 @@ public class Agent implements HandlerFactory, IAgentControl {
_resource.disconnected();
while (true) {
int inProgress = 0;
do {
_shell.getBackoffAlgorithm().waitBeforeRetry();
s_logger.info("Lost connection to the server. Reconnecting....");
int inProgress = 0;
if ((inProgress = _inProgress.get()) > 0) {
inProgress = _inProgress.get();
if (inProgress > 0) {
s_logger.info("Cannot connect because we still have " + inProgress + " commands in progress.");
continue;
}
} while (inProgress > 0);
try {
final SocketChannel sch = SocketChannel.open();
sch.configureBlocking(false);
sch.connect(link.getSocketAddress());
link.connect(sch);
return;
} catch(final IOException e) {
s_logger.error("Unable to establish connection with the server", e);
}
}
_connection.stop();
_connection = new NioClient(
"Agent",
_shell.getHost(),
_shell.getPort(),
_shell.getWorkers(),
this);
do {
s_logger.info("Reconnecting...");
_connection.start();
_shell.getBackoffAlgorithm().waitBeforeRetry();
} while (!_connection.isStartup());
}
public void processStartupAnswer(Answer answer, Response response, Link link) {

View File

@ -174,7 +174,13 @@
<path refid="deps.classpath" />
</path>
<target name="compile-utils" depends="-init" description="Compile the utilities jar that is shared.">
<compile-java jar.name="${utils.jar}" top.dir="${utils.dir}" classpath="utils.classpath" />
<compile-java jar.name="${utils.jar}" top.dir="${utils.dir}" classpath="utils.classpath" >
<include-files>
<fileset dir="${utils.dir}/certs">
<include name="*.keystore" />
</fileset>
</include-files>
</compile-java>
</target>
<property name="api.dir" location="${base.dir}/api" />

View File

@ -25,9 +25,13 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.PreparedStatement;
@ -36,6 +40,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
import java.util.regex.Pattern;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
@ -82,6 +87,7 @@ import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.component.ComponentLocator;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.encoding.Base64.OutputStream;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.NetUtils;
import com.cloud.utils.script.Script;
@ -221,6 +227,9 @@ public class ConfigurationServerImpl implements ConfigurationServer {
}
}
}
// keystore for SSL/TLS connection
updateSSLKeystore();
// store the public and private keys in the database
updateKeyPairs();
@ -369,6 +378,131 @@ public class ConfigurationServerImpl implements ConfigurationServer {
}
}
private String getBase64Keystore(String keystorePath) throws IOException {
byte[] storeBytes = new byte[4094];
int len = 0;
try {
len = new FileInputStream(keystorePath).read(storeBytes);
} catch (EOFException e) {
} catch (Exception e) {
throw new IOException("Cannot read the generated keystore file");
}
if (len > 3000) { // Base64 codec would enlarge data by 1/3, and we have 4094 bytes in database entry at most
throw new IOException("KeyStore is too big for database! Length " + len);
}
byte[] encodeBytes = new byte[len];
System.arraycopy(storeBytes, 0, encodeBytes, 0, len);
return new String(Base64.encodeBase64(encodeBytes));
}
@DB
private void createSSLKeystoreDBEntry(String encodedKeystore) throws IOException {
String insertSQL = "INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) " +
"VALUES ('Hidden','DEFAULT', 'management-server','ssl.keystore', '" + encodedKeystore +"','SSL Keystore for the management servers')";
Transaction txn = Transaction.currentTxn();
try {
PreparedStatement stmt = txn.prepareAutoCloseStatement(insertSQL);
stmt.executeUpdate();
if (s_logger.isDebugEnabled()) {
s_logger.debug("SSL Keystore inserted into database");
}
} catch (SQLException ex) {
s_logger.error("SQL of the SSL Keystore failed", ex);
throw new IOException("SQL of the SSL Keystore failed");
}
}
private void generateDefaultKeystore(String keystorePath) throws IOException {
String cn = "Cloudstack User";
String ou;
try {
ou = InetAddress.getLocalHost().getCanonicalHostName();
String[] group = ou.split("\\.");
// Simple check to see if we got IP Address...
boolean isIPAddress = Pattern.matches("[0-9]$", group[group.length - 1]);
if (isIPAddress) {
ou = "cloud.com";
} else {
ou = group[group.length - 1];
for (int i = group.length - 2; i >= 0 && i >= group.length - 3; i--)
ou = group[i] + "." + ou;
}
} catch (UnknownHostException ex) {
s_logger.info("Fail to get user's domain name. Would use cloud.com. ", ex);
ou = "cloud.com";
}
String o = ou;
String c = "Unknown";
String dname = "cn=" + cn + ", ou=" + ou +", o=" + o + ", c=" + c;
Script script = new Script(true, "keytool", 5000, null);
script.add("-genkey");
script.add("-keystore", keystorePath);
script.add("-storepass", "vmops.com");
script.add("-keypass", "vmops.com");
script.add("-validity", "3650");
script.add("-dname", dname);
String result = script.execute();
if (result != null) {
throw new IOException("Fail to generate certificate!");
}
}
protected void updateSSLKeystore() {
if (s_logger.isInfoEnabled()) {
s_logger.info("Processing updateSSLKeyStore");
}
String dbString = _configDao.getValue("ssl.keystore");
String keystorePath = "/etc/cloud/management/cloud.keystore";
File keystoreFile = new File(keystorePath);
boolean dbExisted = (dbString != null && !dbString.isEmpty());
try {
if (!dbExisted) {
if (!keystoreFile.exists()) {
generateDefaultKeystore(keystorePath);
s_logger.info("Generated SSL keystore.");
}
String base64Keystore = getBase64Keystore(keystorePath);
createSSLKeystoreDBEntry(base64Keystore);
s_logger.info("Stored SSL keystore to database.");
} else if (keystoreFile.exists()) { // and dbExisted
// Check if they are the same one, otherwise override with local keystore
String base64Keystore = getBase64Keystore(keystorePath);
if (base64Keystore.compareTo(dbString) != 0) {
_configDao.update("ssl.keystore", base64Keystore);
s_logger.info("Updated database keystore with local one.");
}
} else { // !keystoreFile.exists() and dbExisted
// Export keystore to local file
byte[] storeBytes = Base64.decodeBase64(dbString);
try {
String tmpKeystorePath = "/tmp/tmpkey";
FileOutputStream fo = new FileOutputStream(tmpKeystorePath);
fo.write(storeBytes);
fo.close();
Script script = new Script(true, "cp", 5000, null);
script.add(tmpKeystorePath);
script.add(keystorePath);
String result = script.execute();
if (result != null) {
throw new IOException();
}
} catch (Exception e) {
throw new IOException("Fail to create keystore file!", e);
}
s_logger.info("Stored database keystore to local.");
}
} catch (Exception ex) {
s_logger.warn("Would use fail-safe keystore to continue.", ex);
}
}
@DB
protected void updateKeyPairs() {
// Grab the SSH key pair and insert it into the database, if it is not present

BIN
utils/certs/cloud.keystore Normal file

Binary file not shown.

View File

@ -28,6 +28,11 @@ import java.nio.channels.SocketChannel;
import java.nio.channels.WritableByteChannel;
import java.util.concurrent.ConcurrentLinkedQueue;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLEngineResult;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLEngineResult.HandshakeStatus;
import org.apache.log4j.Logger;
/**
@ -44,6 +49,8 @@ public class Link {
private Object _attach;
private boolean _readSize;
private SSLEngine _sslEngine;
public Link(InetSocketAddress addr, NioConnection connection) {
_addr = addr;
_connection = connection;
@ -70,6 +77,10 @@ public class Link {
_key = key;
}
public void setSSLEngine(SSLEngine sslEngine) {
_sslEngine = sslEngine;
}
/**
* Static methods for reading from a channel in case
* you need to add a client that doesn't require nio.
@ -190,8 +201,24 @@ public class Link {
}
_readBuffer.flip();
byte[] result = new byte[_readBuffer.limit()];
_readBuffer.get(result);
ByteBuffer appBuf;
SSLSession sslSession = _sslEngine.getSession();
SSLEngineResult engResult;
//TODO may need to adjust the buffer size
appBuf = ByteBuffer.allocate(sslSession.getApplicationBufferSize() + 40);
engResult = _sslEngine.unwrap(_readBuffer, appBuf);
if (engResult.getHandshakeStatus() != HandshakeStatus.FINISHED &&
engResult.getHandshakeStatus() != HandshakeStatus.NOT_HANDSHAKING &&
engResult.getStatus() != SSLEngineResult.Status.OK) {
throw new IOException("SSL: SSLEngine return bad result! " + engResult);
}
byte[] result = new byte[appBuf.position()];
appBuf.flip();
appBuf.get(result);
_readBuffer.clear();
_readSize = true;
@ -258,29 +285,46 @@ public class Link {
return true;
}
data[0].mark();
int remaining = data[0].getInt() + 4;
data[0].reset();
if (remaining > 65535) {
throw new IOException("Fail to send a too big packet! Size: " + remaining);
ByteBuffer pkgBuf;
SSLSession sslSession = _sslEngine.getSession();
SSLEngineResult engResult;
ByteBuffer headBuf = ByteBuffer.allocate(4);
ByteBuffer[] raw_data = new ByteBuffer[data.length - 1];
System.arraycopy(data, 1, raw_data, 0, data.length - 1);
pkgBuf = ByteBuffer.allocate(sslSession.getPacketBufferSize() + 40);
engResult = _sslEngine.wrap(raw_data, pkgBuf);
if (engResult.getHandshakeStatus() != HandshakeStatus.FINISHED &&
engResult.getHandshakeStatus() != HandshakeStatus.NOT_HANDSHAKING &&
engResult.getStatus() != SSLEngineResult.Status.OK) {
throw new IOException("SSL: SSLEngine return bad result! " + engResult);
}
while (remaining > 0) {
int dataRemaining = pkgBuf.position();
int headRemaining = 4;
pkgBuf.flip();
headBuf.putInt(dataRemaining);
headBuf.flip();
while (headRemaining > 0) {
if (s_logger.isTraceEnabled()) {
s_logger.trace("Writing " + remaining);
s_logger.trace("Writing Header " + headRemaining);
}
long count = ch.write(data);
remaining -= count;
long count = ch.write(headBuf);
headRemaining -= count;
}
while (dataRemaining > 0) {
if (s_logger.isTraceEnabled()) {
s_logger.trace("Writing Data " + dataRemaining);
}
long count = ch.write(pkgBuf);
dataRemaining -= count;
}
}
return false;
}
public synchronized void connect(SocketChannel ch) {
_connection.register(SelectionKey.OP_CONNECT, ch, this);
}
public InetSocketAddress getSocketAddress() {
return _addr;
}

View File

@ -23,6 +23,9 @@ import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.nio.channels.SocketChannel;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLContext;
import org.apache.log4j.Logger;
public class NioClient extends NioConnection {
@ -45,23 +48,49 @@ public class NioClient extends NioConnection {
_selector = Selector.open();
SocketChannel sch = SocketChannel.open();
sch.configureBlocking(false);
sch.configureBlocking(true);
s_logger.info("Connecting to " + _host + ":" + _port);
if(_bindAddress != null) {
s_logger.info("Binding outbound interface at " + _bindAddress);
InetSocketAddress addr = new InetSocketAddress(_bindAddress, 0);
sch.socket().bind(addr);
sch.socket().bind(addr);
}
InetSocketAddress addr = new InetSocketAddress(_host, _port);
sch.connect(addr);
try {
sch.connect(addr);
} catch (IOException e) {
_selector.close();
throw e;
}
SSLEngine sslEngine = null;
try {
// Begin SSL handshake in BLOCKING mode
sch.configureBlocking(true);
SSLContext sslContext = initSSLContext(true);
sslEngine = sslContext.createSSLEngine(_host, _port);
sslEngine.setUseClientMode(true);
doHandshake(sch, sslEngine, true);
s_logger.info("SSL: Handshake done");
} catch (Exception e) {
throw new IOException("SSL: Fail to init SSL! " + e);
}
sch.configureBlocking(false);
Link link = new Link(addr, this);
SelectionKey key = sch.register(_selector, SelectionKey.OP_CONNECT);
link.setSSLEngine(sslEngine);
SelectionKey key = sch.register(_selector, SelectionKey.OP_READ);
link.setKey(key);
key.attach(link);
// Notice we've already connected due to the handshake, so let's get the
// remaining task done
Task task = _factory.create(Task.Type.CONNECT, link, null);
_executor.execute(task);
}
@Override

View File

@ -17,15 +17,20 @@
*/
package com.cloud.utils.nio;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.nio.ByteBuffer;
import java.nio.channels.CancelledKeyException;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
import java.security.KeyStore;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@ -35,10 +40,19 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLEngineResult;
import javax.net.ssl.SSLSession;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.SSLEngineResult.HandshakeStatus;
import org.apache.log4j.Logger;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.nio.TrustAllManager;
/**
* NioConnection abstracts the NIO socket operations. The Java implementation
@ -51,6 +65,7 @@ public abstract class NioConnection implements Runnable {
protected Selector _selector;
protected Thread _thread;
protected boolean _isRunning;
protected boolean _isStartup;
protected int _port;
protected List<ChangeRequest> _todos;
protected HandlerFactory _factory;
@ -73,6 +88,16 @@ public abstract class NioConnection implements Runnable {
_thread = new Thread(this, _name + "-Selector");
_isRunning = true;
_thread.start();
// Wait until we got init() done
synchronized(_thread) {
try {
_thread.wait();
} catch (InterruptedException e) {
if (s_logger.isTraceEnabled()) {
s_logger.info("Interrupted start thread ", e);
}
}
}
}
public void stop() {
@ -87,14 +112,25 @@ public abstract class NioConnection implements Runnable {
return _thread.isAlive();
}
public boolean isStartup() {
return _isStartup;
}
public void run() {
try {
init();
} catch (IOException e) {
s_logger.error("Unable to initialize the threads.", e);
return;
}
synchronized(_thread) {
try {
init();
} catch (ConnectException e) {
s_logger.error("Unable to connect to remote");
return;
} catch (IOException e) {
s_logger.error("Unable to initialize the threads.", e);
return;
}
_isStartup = true;
_thread.notifyAll();
}
while (_isRunning) {
try {
_selector.select();
@ -144,22 +180,165 @@ public abstract class NioConnection implements Runnable {
abstract void init() throws IOException;
abstract void registerLink(InetSocketAddress saddr, Link link);
abstract void unregisterLink(InetSocketAddress saddr);
protected SSLContext initSSLContext(boolean isClient) throws Exception {
SSLContext sslContext = null;
KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
KeyStore ks = KeyStore.getInstance("JKS");
TrustManager[] tms;
if (!isClient) {
char[] passphrase = "vmops.com".toCharArray();
String keystorePath = "/etc/cloud/management/cloud.keystore";
if (new File(keystorePath).exists()) {
ks.load(new FileInputStream(keystorePath), passphrase);
} else {
s_logger.warn("SSL: Fail to find the generated keystore. Loading fail-safe one to continue.");
ks.load(NioConnection.class.getResourceAsStream("/cloud.keystore"), passphrase);
}
kmf.init(ks, passphrase);
tmf.init(ks);
tms = tmf.getTrustManagers();
} else {
ks.load(null, null);
kmf.init(ks, null);
tms = new TrustManager[1];
tms[0] = new TrustAllManager();
}
sslContext = SSLContext.getInstance("TLS");
sslContext.init(kmf.getKeyManagers(), tms, null);
s_logger.info("SSL: SSLcontext has been initialized");
return sslContext;
}
protected void doHandshake(SocketChannel ch, SSLEngine sslEngine,
boolean isClient) throws IOException {
s_logger.info("SSL: begin Handshake, isClient: " + isClient);
SSLEngineResult engResult;
SSLSession sslSession = sslEngine.getSession();
HandshakeStatus hsStatus;
ByteBuffer in_pkgBuf =
ByteBuffer.allocate(sslSession.getPacketBufferSize() + 40);
ByteBuffer in_appBuf =
ByteBuffer.allocate(sslSession.getApplicationBufferSize() + 40);
ByteBuffer out_pkgBuf =
ByteBuffer.allocate(sslSession.getPacketBufferSize() + 40);
ByteBuffer out_appBuf =
ByteBuffer.allocate(sslSession.getApplicationBufferSize() + 40);
int count;
if (isClient) {
hsStatus = SSLEngineResult.HandshakeStatus.NEED_WRAP;
} else {
hsStatus = SSLEngineResult.HandshakeStatus.NEED_UNWRAP;
}
while (hsStatus != SSLEngineResult.HandshakeStatus.FINISHED) {
if (s_logger.isTraceEnabled()) {
s_logger.info("SSL: Handshake status " + hsStatus);
}
engResult = null;
if (hsStatus == SSLEngineResult.HandshakeStatus.NEED_WRAP) {
out_pkgBuf.clear();
out_appBuf.clear();
out_appBuf.put("Hello".getBytes());
engResult = sslEngine.wrap(out_appBuf, out_pkgBuf);
out_pkgBuf.flip();
int remain = out_pkgBuf.limit();
while (remain != 0) {
remain -= ch.write(out_pkgBuf);
if (remain < 0) {
throw new IOException("Too much bytes sent?");
}
}
} else if (hsStatus == SSLEngineResult.HandshakeStatus.NEED_UNWRAP) {
in_appBuf.clear();
// One packet may contained multiply operation
if (in_pkgBuf.position() == 0 || !in_pkgBuf.hasRemaining()) {
in_pkgBuf.clear();
count = ch.read(in_pkgBuf);
if (count == -1) {
throw new IOException("Connection closed with -1 on reading size.");
}
in_pkgBuf.flip();
}
engResult = sslEngine.unwrap(in_pkgBuf, in_appBuf);
ByteBuffer tmp_pkgBuf =
ByteBuffer.allocate(sslSession.getPacketBufferSize() + 40);
while (engResult.getStatus() == SSLEngineResult.Status.BUFFER_UNDERFLOW) {
// We need more packets to complete this operation
if (s_logger.isTraceEnabled()) {
s_logger.info("SSL: Buffer overflowed, getting more packets");
}
tmp_pkgBuf.clear();
count = ch.read(tmp_pkgBuf);
tmp_pkgBuf.flip();
in_pkgBuf.mark();
in_pkgBuf.position(in_pkgBuf.limit());
in_pkgBuf.limit(in_pkgBuf.limit() + tmp_pkgBuf.limit());
in_pkgBuf.put(tmp_pkgBuf);
in_pkgBuf.reset();
in_appBuf.clear();
engResult = sslEngine.unwrap(in_pkgBuf, in_appBuf);
}
} else if (hsStatus == SSLEngineResult.HandshakeStatus.NEED_TASK) {
Runnable run;
while ((run = sslEngine.getDelegatedTask()) != null) {
if (s_logger.isTraceEnabled()) {
s_logger.info("SSL: Running delegated task!");
}
run.run();
}
} else if (hsStatus == SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING) {
throw new IOException("NOT a handshaking!");
}
if (engResult != null && engResult.getStatus() != SSLEngineResult.Status.OK) {
throw new IOException("Fail to handshake! " + engResult.getStatus());
}
if (engResult != null)
hsStatus = engResult.getHandshakeStatus();
else
hsStatus = sslEngine.getHandshakeStatus();
}
}
protected void accept(SelectionKey key) throws IOException {
ServerSocketChannel serverSocketChannel = (ServerSocketChannel)key.channel();
SocketChannel socketChannel = serverSocketChannel.accept();
Socket socket = socketChannel.socket();
socketChannel.configureBlocking(false);
socket.setKeepAlive(true);
if (s_logger.isTraceEnabled()) {
s_logger.trace("Connection accepted for " + socket);
}
// Begin SSL handshake in BLOCKING mode
socketChannel.configureBlocking(true);
SSLEngine sslEngine = null;
try {
SSLContext sslContext = initSSLContext(false);
sslEngine = sslContext.createSSLEngine();
sslEngine.setUseClientMode(false);
sslEngine.setNeedClientAuth(false);
doHandshake(socketChannel, sslEngine, false);
s_logger.info("SSL: Handshake done");
} catch (Exception e) {
throw new IOException("SSL: Fail to init SSL! " + e);
}
socketChannel.configureBlocking(false);
InetSocketAddress saddr = (InetSocketAddress)socket.getRemoteSocketAddress();
Link link = new Link(saddr, this);
link.setSSLEngine(sslEngine);
link.setKey(socketChannel.register(key.selector(), SelectionKey.OP_READ, link));
Task task = _factory.create(Task.Type.CONNECT, link, null);
registerLink(saddr, link);

View File

@ -0,0 +1,43 @@
/**
* Copyright (C) 2011 Cloud.com, Inc. All rights reserved.
*
* This software is licensed under the GNU General Public License v3 or later.
*
* It is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
package com.cloud.utils.nio;
public class TrustAllManager implements javax.net.ssl.TrustManager, javax.net.ssl.X509TrustManager {
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return null;
}
public boolean isServerTrusted(java.security.cert.X509Certificate[] certs) {
return true;
}
public boolean isClientTrusted(java.security.cert.X509Certificate[] certs) {
return true;
}
public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType)
throws java.security.cert.CertificateException {
return;
}
public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType)
throws java.security.cert.CertificateException {
return;
}
}