diff --git a/src/main/java/org/spiderflow/selenium/executor/function/extension/SeleniumResponseFunctionExtension.java b/src/main/java/org/spiderflow/selenium/executor/function/extension/SeleniumResponseFunctionExtension.java index 44e5326181aea6e6aea8d26c30f9e772c614591d..8834dfe53af9f7c4dd71921cc602aceaf2adea07 100644 --- a/src/main/java/org/spiderflow/selenium/executor/function/extension/SeleniumResponseFunctionExtension.java +++ b/src/main/java/org/spiderflow/selenium/executor/function/extension/SeleniumResponseFunctionExtension.java @@ -22,7 +22,7 @@ public class SeleniumResponseFunctionExtension implements FunctionExtension { } @Comment("根据css选择器提取请求结果") - @Example("${resp.selector('div > a')}") + @Example("${sele.selector('div > a')}") public static WebElementWrapper selector(SeleniumResponse response, String css) { try { return new WebElementWrapper(response, response.getDriver().findElement(By.cssSelector(css))); @@ -32,7 +32,7 @@ public class SeleniumResponseFunctionExtension implements FunctionExtension { } @Comment("根据css选择器提取请求结果") - @Example("${resp.selector('div > a')}") + @Example("${sele.selector('div > a')}") public static WebElements selectors(SeleniumResponse response, String css) { try { return new WebElements(response, response.getDriver().findElements(By.cssSelector(css))); @@ -42,7 +42,7 @@ public class SeleniumResponseFunctionExtension implements FunctionExtension { } @Comment("根据xpath在请求结果中查找") - @Example("${resp.xpaths('//a')}") + @Example("${sele.xpaths('//a')}") public static WebElements xpaths(SeleniumResponse response, String xpath) { try { return new WebElements(response, response.getDriver().findElements(By.xpath(xpath))); @@ -52,7 +52,7 @@ public class SeleniumResponseFunctionExtension implements FunctionExtension { } @Comment("根据xpath在请求结果中查找") - @Example("${resp.xpath('//a')}") + @Example("${sele.xpath('//a')}") public static WebElement xpath(SeleniumResponse response, String xpath) { try { return new WebElementWrapper(response, response.getDriver().findElement(By.xpath(xpath))); @@ -62,13 +62,13 @@ public class SeleniumResponseFunctionExtension implements FunctionExtension { } @Comment("执行js") - @Example("${resp.executeScript(\"document.write('hello spider-flow !')\")}") + @Example("${sele.executeScript(\"document.write('hello spider-flow !')\")}") public static Object executeScript(SeleniumResponse response, String script) { return executeScript(response, script, null); } @Comment("执行js") - @Example("${resp.executeScript(\"document.write('hello '+arguments[0]+' !')\",\"spider-flow\")}") + @Example("${sele.executeScript(\"document.write('hello '+arguments[0]+' !')\",\"spider-flow\")}") public static Object executeScript(SeleniumResponse response, String script, List arguments) { JavascriptExecutor executor = null; try { @@ -79,4 +79,11 @@ public class SeleniumResponseFunctionExtension implements FunctionExtension { return executor.executeScript(script, arguments); } + @Comment("跳转URL") + @Example("${sele.toUrl(newUrl)}") + public static SeleniumResponse toUrl(SeleniumResponse response, String newUrl) { + response.getDriver().get(newUrl); + return response; + } + } diff --git a/src/main/java/org/spiderflow/selenium/executor/shape/SeleniumExecutor.java b/src/main/java/org/spiderflow/selenium/executor/shape/SeleniumExecutor.java index d7a089ed94faaa70d46a400cae9e2b27b5f01508..06b238cb108969a33d61e4de50c0a9f6d47dc9ce 100644 --- a/src/main/java/org/spiderflow/selenium/executor/shape/SeleniumExecutor.java +++ b/src/main/java/org/spiderflow/selenium/executor/shape/SeleniumExecutor.java @@ -3,6 +3,7 @@ package org.spiderflow.selenium.executor.shape; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.math.NumberUtils; +import org.openqa.selenium.Cookie; import org.openqa.selenium.Proxy; import org.openqa.selenium.WebDriver; import org.openqa.selenium.chrome.ChromeDriver; @@ -13,6 +14,7 @@ import org.openqa.selenium.remote.DesiredCapabilities; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.spiderflow.ExpressionEngine; +import org.spiderflow.context.CookieContext; import org.spiderflow.context.SpiderContext; import org.spiderflow.executor.ShapeExecutor; import org.spiderflow.model.Shape; @@ -24,6 +26,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; +import java.net.URL; +import java.util.Calendar; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -46,6 +50,8 @@ public class SeleniumExecutor implements ShapeExecutor { public static final String PROXY = "proxy"; + public static final String COOKIE_AUTO_SET = "cookie-auto-set"; + @Autowired private List driverProviders; @@ -92,6 +98,12 @@ public class SeleniumExecutor implements ShapeExecutor { context.error("设置代理出错，异常信息：{}", e); } } + Object oldResp = variables.get("sele"); + //REVIEW 一个任务流中只能有一个Driver，在页面跳转操作可以使用sele.toUrl，打来其他Driver时，原页面会关闭 + if(oldResp instanceof SeleniumResponse){ + SeleniumResponse oldResponse = (SeleniumResponse) oldResp; + oldResponse.quit(); + } WebDriver driver = null; try { String url = engine.execute(node.getStringJsonValue(URL), variables).toString(); @@ -99,17 +111,32 @@ public class SeleniumExecutor implements ShapeExecutor { driver = providerMap.get(driverType).getWebDriver(node, proxy); driver.manage().timeouts().pageLoadTimeout(NumberUtils.toInt(node.getStringJsonValue(PAGE_LOAD_TIMEOUT), 60 * 1000), TimeUnit.MILLISECONDS); driver.manage().timeouts().implicitlyWait(NumberUtils.toInt(node.getStringJsonValue(IMPLICITLY_WAIT_TIMEOUT), 3 * 1000), TimeUnit.MILLISECONDS); + //设置自动管理的Cookie + boolean cookieAutoSet = !"0".equals(node.getStringJsonValue(COOKIE_AUTO_SET)); + CookieContext cookieContext = context.getCookieContext(); + //初始化打开浏览器 + driver.get(url); + //设置浏览器Cookies环境 + if(cookieAutoSet){ + driver.manage().deleteAllCookies(); + URL tempUrl = new URL(url); + Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.MONTH, 1); + for (Map.Entry item : cookieContext.entrySet()) { + Cookie cookie = new Cookie(item.getKey(), item.getValue(), tempUrl.getHost(), "/", calendar.getTime() , false, false); + driver.manage().addCookie(cookie); + } + context.debug("自动设置Cookie：{}", cookieContext); + } + //访问跳转url网站 driver.get(url); SeleniumResponse response = new SeleniumResponse(driver); SeleniumResponseHolder.add(context, response); - Object oldResp = variables.get("resp"); - - //TODO 会造成其他线程无法使用resp,但是也不能长期不释放...综合考虑在请求时，如果变量中有SeleniumResponse，就释放掉之前的driver - if(oldResp instanceof SeleniumResponse){ - SeleniumResponse oldResponse = (SeleniumResponse) oldResp; - oldResponse.quit(); + if(cookieAutoSet){ + Map cookies = response.getCookies(); + cookieContext.putAll(cookies); } - variables.put("resp", response); + variables.put("sele", response); } catch (Exception e) { context.error("请求出错，异常信息：{}", e); if (driver != null) { diff --git a/src/main/resources/static/resources/templates/selenium.html b/src/main/resources/static/resources/templates/selenium.html index da29d78c64c596aa620947e099ff0cc980d69150..c7fda954d12272e199187701bd4e8c3e104fb94d 100644 --- a/src/main/resources/static/resources/templates/selenium.html +++ b/src/main/resources/static/resources/templates/selenium.html @@ -54,6 +54,12 @@
+
+ +
+ +
+