From ba457095cc09dbf553822f091f6693fa1f5a60c0 Mon Sep 17 00:00:00 2001 From: gitee-bot Date: Fri, 20 Jun 2025 01:40:43 +0000 Subject: [PATCH] Update README.md --- README.md | 1731 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 1528 insertions(+), 203 deletions(-) diff --git a/README.md b/README.md index b94be45..385621b 100644 --- a/README.md +++ b/README.md @@ -1,213 +1,1538 @@ -![banner.png](images/banner3.png) -

- JDK - Downloads - Downloads - - +# qData 平台文档 - - -

+## 🌈 平台简介 +qData 是一个数据处理与调度平台，提供 ETL（抽取、转换、加载）任务的创建、执行、调度与监控功能。它支持多种数据源（如 Kafka、MySQL、Hive 等），并集成了 Quartz 定时任务调度框架，实现灵活的任务管理。 -

- 📖简体中文 | 📖English -

+平台基于 Spring Boot、MyBatis Plus、Redis、RabbitMQ、Quartz 等主流技术栈构建，具备良好的可扩展性与稳定性，适用于企业级数据集成与治理场景。 +## 🍱 使用场景 +- 数据集成：从不同数据源抽取数据并加载到目标数据库。 +- 数据清洗与转换：通过内置规则对数据进行标准化、脱敏、字段映射等操作。 +- 任务调度管理：定时执行 ETL 任务，支持任务启停、状态监控、日志记录。 +- 多租户与权限控制：支持基于角色的数据权限与操作权限管理。 +- 数据可视化：提供任务执行状态与日志的可视化展示。 -## 🌈平台简介 -**千数平台（qData）** 是一款一**站式开源数据中台**，覆盖**中台基础建设、数据治理、数据开发、监控告警、数据服务**与**数据可视化**等核心能力，致力于高效赋能数据前台，为各类业务系统提供**统一、高效、安全**的数据支撑服务。 +## 💡 优势 +- **多数据源支持**：支持 Kafka、MySQL、Oracle、Hive、DM ://qiantong/qdata/spark/etl/utils/IDGeneratorUtils.java + - public static long getLongId() + - public static String getStringId() + - public static void main(String[] args) -✨✨✨**演示地址**✨✨✨ 点击跳转 ，账号：qData 密码：qData123 +### qdata-etl/src/main/java/tech/qiantong/qdata/spark/etl/utils/LogUtils.java +public class LogUtils + - public static String createLogPath(String resourceUrl, JSONObject nodeJson) + - public static void writeLog(String logPath, String meesage) -✨✨✨**演示视频**✨✨✨ https://www.bilibili.com/video/BV1d6TxziEdX +### qdata-etl/src/main/java/tech/qiantong/qdata/spark/etl/utils/RabbitmqUtils.java +public class RabbitmqUtils + - public static Boolean convertAndSend(JSONObject config, String exchange, String routingKey, Object object) + - public static void main(String[] args) -## 🍱 使用场景 +### qdata-etl/src/main/java/tech/qiantong/qdata/spark/etl/utils/SnowflakeIdHelper.java +public class SnowflakeIdHelper + - private final long twepoch + - private final long workerIdBits + - private final long datacenterIdBits + - private final long maxWorkerId + - private final long maxDatacenterId + - private final long sequenceBits + - private final long workerIdShift + - private final long datacenterIdShift + - private final long timestampLeftShift + - private final long sequenceMask + - private long workerId + - private long datacenterId + - private long sequence + - private long lastTimestamp + - private static SnowflakeIdHelper instance + - public static SnowflakeIdHelper getInstance(long workerId, long datacenterId) + - public SnowflakeIdHelper(long workerId, long datacenterId) + - public synchronized long nextId() + - protected long tilNextMillis(long lastTimestamp) + - protected long timeGen() -| 场景 | 描述 | -|----------------------|----------------------------------------------------------------------| -| 🔗 **多源数据整合** | 整合业务系统、IoT、第三方等数据，实现统一采集、清洗与治理，构建标准数据资产。 | -| 🌐 **统一数据服务** | 将治理数据通过 API 或数据集方式提供，支撑系统共享调用，提升复用效率。 | -| 📈 **决策分析支撑** | 提供统一数据视图和分析能力，助力业务分析、报表生成及科学决策。 | +### qdata-etl/src/main/java/tech/qiantong/qdata/spark/etl/utils/db/DBUtils.java +public class DBUtils + - public static DbQueryProperty buildJobDatasource(Map datasource) + - public static Map getDbOptions(JSONObject parameter) + - public static void init() -## 💡 优势 +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/controller/BaseController.java +public class BaseController + - protected final Logger logger + - @InitBinder + public void initBinder(WebDataBinder binder) + - protected void startPage() + - protected void startOrderBy() + - protected void clearPage() + - @SuppressWarnings({ "rawtypes", "unchecked" }) + protected TableDataInfo getDataTable(List list) + - public AjaxResult success() + - public AjaxResult error() + - public AjaxResult success(String message) + - public AjaxResult success(Object data) + - public AjaxResult error(String message) + - public AjaxResult warn(String message) + - protected AjaxResult toAjax(int rows) + - protected AjaxResult toAjax(boolean result) + - public String redirect(String url) + - public LoginUser getLoginUser() + - public Long getUserId() + - public Long getDeptId() + - public String getUsername() + - public String getNickName() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/AjaxResult.java +public class AjaxResult extends HashMap + - private static final long serialVersionUID + - public static final String CODE_TAG + - public static final String MSG_TAG + - public static final String DATA_TAG + - public AjaxResult() + - public AjaxResult(int code, String msg) + - public AjaxResult(int code, String msg, Object data) + - public static AjaxResult success() + - public static AjaxResult success(Object data) + - public static AjaxResult success(String msg) + - public static AjaxResult success(String msg, Object data) + - public static AjaxResult warn(String msg) + - public static AjaxResult warn(String msg, Object data) + - public static AjaxResult error() + - public static AjaxResult error(String msg) + - public static AjaxResult error(String msg, Object data) + - public static AjaxResult error(int code, String msg) + - public boolean isSuccess() + - public boolean isWarn() + - public boolean isError() + - @Override + public AjaxResult put(String key, Object value) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/BaseEntity.java +public class BaseEntity implements Serializable + - private static final long serialVersionUID + - @JsonIgnore + @TableField(exist + - @Schema(description + - @Schema(description + - @TableField(fill + - @Schema(description + - @JsonInclude(JsonInclude.Include.NON_EMPTY) + @TableField(exist + - public String getSearchValue() + - public void setSearchValue(String searchValue) + - public String getCreateBy() + - public void setCreateBy(String createBy) + - public Date getCreateTime() + - public void setCreateTime(Date createTime) + - public String getUpdateBy() + - public void setUpdateBy(String updateBy) + - public Date getUpdateTime() + - public void setUpdateTime(Date updateTime) + - public Long getCreatorId() + - public void setCreatorId(Long creatorId) + - public Long getUpdatorId() + - public void setUpdatorId(Long updaterId) + - public String getRemark() + - public void setRemark(String remark) + - public Map getParams() + - public Object getParamByKey(String key) + - public void setParams(Map params) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/CommonResult.java +@Schema(description = "通用返回") +@Data +public class CommonResult implements Serializable + - private static final long serialVersionUID + - @Schema(description + - @Schema(description + - @Schema(description + - public static CommonResult error(CommonResult result) + - public static CommonResult error(Integer code, String message) + - public static CommonResult error(ErrorCode errorCode) + - public static CommonResult success(T data) + - public static CommonResult toAjax(T data) + - public static boolean isSuccess(Integer code) + - @JsonIgnore // 避免 jackson 序列化 + public boolean isSuccess() + - @JsonIgnore // 避免 jackson 序列化 + public boolean isError() + - public void checkError() throws ServiceException + - @JsonIgnore // 避免 jackson 序列化 + public T getCheckedData() + - public static CommonResult error(ServiceException serviceException) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/R.java +public class R implements Serializable + - private static final long serialVersionUID + - public static final int SUCCESS + - public static final int FAIL + - private int code + - private String msg + - private T data + - public static R ok() + - public static R ok(T data) + - public static R ok(T data, String msg) + - public static R fail() + - public static R fail(String msg) + - public static R fail(T data) + - public static R fail(T data, String msg) + - public static R fail(int code, String msg) + - private static R restResult(T data, int code, String msg) + - public int getCode() + - public void setCode(int code) + - public String getMsg() + - public void setMsg(String msg) + - public T getData() + - public void setData(T data) + - public static Boolean isError(R ret) + - public static Boolean isSuccess(R ret) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/ReturnT.java +public class ReturnT implements Serializable + - public static final long serialVersionUID + - public static final int SUCCESS_CODE + - public static final int FAIL_CODE + - public static final ReturnT SUCCESS + - public static final ReturnT FAIL + - private int code + - private String msg + - private T content + - public ReturnT() + - public ReturnT(int code, String msg) + - public ReturnT(T content) + - public int getCode() + - public void setCode(int code) + - public String getMsg() + - public void setMsg(String msg) + - public T getContent() + - public void setContent(T content) + - @Override + public String toString() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/TreeEntity.java +public class TreeEntity extends BaseEntity + - private static final long serialVersionUID + - private String parentName + - private Long parentId + - private Integer orderNum + - private String ancestors + - private List children + - public String getParentName() + - public void setParentName(String parentName) + - public Long getParentId() + - public void setParentId(Long parentId) + - public Integer getOrderNum() + - public void setOrderNum(Integer orderNum) + - public String getAncestors() + - public void setAncestors(String ancestors) + - public List getChildren() + - public void setChildren(List children) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/TreeSelect.java +public class TreeSelect implements Serializable + - private static final long serialVersionUID + - private Long id + - private String label + - @JsonInclude(JsonInclude.Include.NON_EMPTY) + private List children + - public TreeSelect() + - public TreeSelect(SysDept dept) + - public TreeSelect(SysMenu menu) + - public TreeSelect(RpDept rpDeptDO) + - public TreeSelect(RpAreaDict rpDeptDO) + - public Long getId() + - public void setId(Long id) + - public String getLabel() + - public void setLabel(String label) + - public List getChildren() + - public void setChildren(List children) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/entity/RpAreaDict.java +@Data +public class RpAreaDict extends BaseEntity + - private static final long serialVersionUID + - private Long id + - private String name + - private String code + - private Long parentId + - private Long sortOrder + - private String type + - private Boolean validFlag + - @TableLogic + private Boolean delFlag + - private List children + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/entity/RpDept.java +@Data +public class RpDept extends BaseEntity + - private static final long serialVersionUID + - private Long deptId + - private Long parentId + - private String ancestors + - private String deptName + - private String deptCode + - private String deptType + - private String orderNum + - private String leader + - private String phone + - private String email + - private String status + - private Boolean validFlag + - @TableLogic + private Boolean delFlag + - private List children + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/entity/SysDept.java +public class SysDept extends BaseEntity + - private static final long serialVersionUID + - private Long deptId + - private Long parentId + - private String ancestors + - private String deptName + - private Integer orderNum + - private String leader + - private String phone + - private String email + - private String status + - private String delFlag + - private String parentName + - private List children + - public Long getDeptId() + - public void setDeptId(Long deptId) + - public Long getParentId() + - public void setParentId(Long parentId) + - public String getAncestors() + - public void setAncestors(String ancestors) + - @NotBlank(message = "部门名称不能为空") + @Size(min = 0, max = 30, message = "部门名称长度不能超过30个字符") + public String getDeptName() + - public void setDeptName(String deptName) + - @NotNull(message = "显示顺序不能为空") + public Integer getOrderNum() + - public void setOrderNum(Integer orderNum) + - public String getLeader() + - public void setLeader(String leader) + - @Size(min = 0, max = 11, message = "联系电话长度不能超过11个字符") + public String getPhone() + - public void setPhone(String phone) + - @Email(message = "邮箱格式不正确") + @Size(min = 0, max = 50, message = "邮箱长度不能超过50个字符") + public String getEmail() + - public void setEmail(String email) + - public String getStatus() + - public void setStatus(String status) + - public String getDelFlag() + - public void setDelFlag(String delFlag) + - public String getParentName() + - public void getParentName(String parentName) + - public List getChildren() + - public void setChildren(List children) + - @Override + public String toString() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/entity/SysDictData.java +public class SysDictData extends BaseEntity + - private static final long serialVersionUID + - @Excel(name + - @Excel(name + - @Excel(name + - @Excel(name + - @Excel(name + - private String cssClass + - private String listClass + - @Excel(name + - @Excel(name + - public Long getDictCode() + - public void setDictCode(Long dictCode) + - public Long getDictSort() + - public void setDictSort(Long dictSort) + - @NotBlank(message = "字典标签不能为空") + @Size(min = 0, max = 100, message = "字典标签长度不能超过100个字符") + public String getDictLabel() + - public void setDictLabel(String dictLabel) + - @NotBlank(message = "字典键值不能为空") + @Size(min = 0, max = 100, message = "字典键值长度不能超过100个字符") + public String getDictValue() + - public void setDictValue(String dictValue) + - @NotBlank(message = "字典类型不能为空") + @Size(min = 0, max = 100, message = "字典类型长度不能超过100个字符") + public String getDictType() + - public void setDictType(String dictType) + - @Size(min = 0, max = 100, message = "样式属性长度不能超过100个字符") + public String getCssClass() + - public void setCssClass(String cssClass) + - public String getListClass() + - public void setListClass(String listClass) + - public boolean getDefault() + - public String getIsDefault() + - public void setIsDefault(String isDefault) + - public String getStatus() + - public void setStatus(String status) + - @Override + public String toString() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/entity/SysDictType.java +public class SysDictType extends BaseEntity + - private static final long serialVersionUID + - @Excel(name + - @Excel(name + - @Excel(name + - @Excel(name + - private List sysDictData + - public List getSysDictData() + - public void setSysDictData(List sysDictData) + - public Long getDictId() + - public void setDictId(Long dictId) + - @NotBlank(message = "字典名称不能为空") + @Size(min = 0, max = 100, message = "字典类型名称长度不能超过100个字符") + public String getDictName() + - public void setDictName(String dictName) + - @NotBlank(message = "字典类型不能为空") + @Size(min = 0, max = 100, message = "字典类型类型长度不能超过100个字符") + @Pattern(regexp = "^[a-z][a-z0-9_]*$", message = "字典类型必须以字母开头，且只能为（小写字母，数字，下滑线）") + public String getDictType() + - public void setDictType(String dictType) + - public String getStatus() + - public void setStatus(String status) + - @Override + public String toString() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/entity/SysMenu.java +public class SysMenu extends BaseEntity + - private static final long serialVersionUID + - private Long menuId + - private String menuName + - private String parentName + - private Long parentId + - private Integer orderNum + - private String path + - private String component + - private String query + - private String routeName + - private String isFrame + - private String isCache + - private String menuType + - private String visible + - private String status + - private String perms + - private String icon + - private Long projectId + - private List children + - public Long getMenuId() + - public void setMenuId(Long menuId) + - @NotBlank(message = "菜单名称不能为空") + @Size(min = 0, max = 50, message = "菜单名称长度不能超过50个字符") + public String getMenuName() + - public void setMenuName(String menuName) + - public String getParentName() + - public void setParentName(String parentName) + - public Long getParentId() + - public void setParentId(Long parentId) + - @NotNull(message = "显示顺序不能为空") + public Integer getOrderNum() + - public void setOrderNum(Integer orderNum) + - @Size(min = 0, max = 200, message = "路由地址不能超过200个字符") + public String getPath() + - public void setPath(String path) + - @Size(min = 0, max = 200, message = "组件路径不能超过255个字符") + public String getComponent() + - public void setComponent(String component) + - public String getQuery() + - public void setQuery(String query) + - public String getRouteName() + - public void setRouteName(String routeName) + - public String getIsFrame() + - public void setIsFrame(String isFrame) + - public String getIsCache() + - public void setIsCache(String isCache) + - @NotBlank(message = "菜单类型不能为空") + public String getMenuType() + - public void setMenuType(String menuType) + - public String getVisible() + - public void setVisible(String visible) + - public String getStatus() + - public void setStatus(String status) + - @Size(min = 0, max = 100, message = "权限标识长度不能超过100个字符") + public String getPerms() + - public void setPerms(String perms) + - public String getIcon() + - public void setIcon(String icon) + - public List getChildren() + - public void setChildren(List children) + - public Long getProjectId() + - public void setProjectId(Long projectId) + - @Override + public String toString() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/entity/SysRole.java +public class SysRole extends BaseEntity + - private static final long serialVersionUID + - @Excel(name + - @Excel(name + - @Excel(name + - @Excel(name + - @Excel(name + - private boolean menuCheckStrictly + - private boolean deptCheckStrictly + - @Excel(name + - private String delFlag + - private boolean flag + - private Long[] menuIds + - private Long[] deptIds + - private Set permissions + - private Long oldRoleId + - public Long getOldRoleId() + - public void setOldRoleId(Long oldRoleId) + - private Long projectId + - public Long getProjectId() + - public void setProjectId(Long projectId) + - public SysRole() + - public SysRole(Long roleId) + - public Long getRoleId() + - public void setRoleId(Long roleId) + - public boolean isAdmin() + - public static boolean isAdmin(Long roleId) + - @NotBlank(message = "角色名称不能为空") + @Size(min = 0, max = 30, message = "角色名称长度不能超过30个字符") + public String getRoleName() + - public void setRoleName(String roleName) + - @NotBlank(message = "权限字符不能为空") + @Size(min = 0, max = 100, message = "权限字符长度不能超过100个字符") + public String getRoleKey() + - public void setRoleKey(String roleKey) + - @NotNull(message = "显示顺序不能为空") + public Integer getRoleSort() + - public void setRoleSort(Integer roleSort) + - public String getDataScope() + - public void setDataScope(String dataScope) + - public boolean isMenuCheckStrictly() + - public void setMenuCheckStrictly(boolean menuCheckStrictly) + - public boolean isDeptCheckStrictly() + - public void setDeptCheckStrictly(boolean deptCheckStrictly) + - public String getStatus() + - public void setStatus(String status) + - public String getDelFlag() + - public void setDelFlag(String delFlag) + - public boolean isFlag() + - public void setFlag(boolean flag) + - public Long[] getMenuIds() + - public void setMenuIds(Long[] menuIds) + - public Long[] getDeptIds() + - public void setDeptIds(Long[] deptIds) + - public Set getPermissions() + - public void setPermissions(Set permissions) + - @Override + public String toString() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/entity/SysUser.java +public class SysUser extends BaseEntity + - private static final long serialVersionUID + - @Excel(name + - private String authId + - @Excel(name + - @Excel(name + - @Excel(name + - @Excel(name + - @Excel(name + - @Excel(name + - private String avatar + - private String password + - @Excel(name + - private String delFlag + - @Excel(name + - @Excel(name + - @Excels({ + @Excel(name + - private List roles + - private Long[] roleIds + - private Long[] postIds + - private List userIdList + - private Long roleId + - public SysUser() + - public SysUser(Long userId) + - public Long getUserId() + - public void setUserId(Long userId) + - public String getAuthId() + - public void setAuthId(String authId) + - public boolean isAdmin() + - public static boolean isAdmin(Long userId) + - public Long getDeptId() + - public void setDeptId(Long deptId) + - public List getUserIdList() + - public void setUserIdList(List userIdList) + - @Xss(message = "用户昵称不能包含脚本字符") + @Size(min = 0, max = 30, message = "用户昵称长度不能超过30个字符") + public String getNickName() + - public void setNickName(String nickName) + - @Xss(message = "用户账号不能包含脚本字符") + @NotBlank(message = "用户账号不能为空") + @Size(min = 0, max = 30, message = "用户账号长度不能超过30个字符") + public String getUserName() + - public void setUserName(String userName) + - @Email(message = "邮箱格式不正确") + @Size(min = 0, max = 50, message = "邮箱长度不能超过50个字符") + public String getEmail() + - public void setEmail(String email) + - @Size(min = 0, max = 11, message = "手机号码长度不能超过11个字符") + public String getPhonenumber() + - public void setPhonenumber(String phonenumber) + - public String getSex() + - public void setSex(String sex) + - public String getAvatar() + - public void setAvatar(String avatar) + - public String getPassword() + - public void setPassword(String password) + - public String getStatus() + - public void setStatus(String status) + - public String getDelFlag() + - public void setDelFlag(String delFlag) + - public String getLoginIp() + - public void setLoginIp(String loginIp) + - public Date getLoginDate() + - public void setLoginDate(Date loginDate) + - public SysDept getDept() + - public void setDept(SysDept dept) + - public List getRoles() + - public void setRoles(List roles) + - public Long[] getRoleIds() + - public void setRoleIds(Long[] roleIds) + - public Long[] getPostIds() + - public void setPostIds(Long[] postIds) + - public Long getRoleId() + - public void setRoleId(Long roleId) + - public Boolean comparePwd(String pwd) + - @Override + public String toString() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/model/LoginBody.java +public class LoginBody + - private String username + - private String password + - private String code + - private String uuid + - public String getUsername() + - public void setUsername(String username) + - public String getPassword() + - public void setPassword(String password) + - public String getCode() + - public void setCode(String code) + - public String getUuid() + - public void setUuid(String uuid) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/model/LoginUser.java +public class LoginUser implements UserDetails + - private static final long serialVersionUID + - private Long userId + - private Long deptId + - private String token + - private Long loginTime + - private Long expireTime + - private String ipaddr + - private String loginLocation + - private String browser + - private String os + - private Set permissions + - private SysUser user + - public LoginUser() + - public LoginUser(SysUser user, Set permissions) + - public LoginUser(Long userId, Long deptId, SysUser user, Set permissions) + - public Long getUserId() + - public void setUserId(Long userId) + - public Long getDeptId() + - public void setDeptId(Long deptId) + - public String getToken() + - public void setToken(String token) + - @JSONField(serialize = false) + @Override + public String getPassword() + - @Override + public String getUsername() + - @JSONField(serialize = false) + @Override + public boolean isAccountNonExpired() + - @JSONField(serialize = false) + @Override + public boolean isAccountNonLocked() + - @JSONField(serialize = false) + @Override + public boolean isCredentialsNonExpired() + - @JSONField(serialize = false) + @Override + public boolean isEnabled() + - public Long getLoginTime() + - public void setLoginTime(Long loginTime) + - public String getIpaddr() + - public void setIpaddr(String ipaddr) + - public String getLoginLocation() + - public void setLoginLocation(String loginLocation) + - public String getBrowser() + - public void setBrowser(String browser) + - public String getOs() + - public void setOs(String os) + - public Long getExpireTime() + - public void setExpireTime(Long expireTime) + - public Set getPermissions() + - public void setPermissions(Set permissions) + - public SysUser getUser() + - public void setUser(SysUser user) + - @Override + public Collection getAuthorities() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/domain/model/RegisterBody.java +public class RegisterBody extends LoginBody + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/page/PageDomain.java +public class PageDomain + - private Integer pageNum + - private Integer pageSize + - private String orderByColumn + - private String isAsc + - private Boolean reasonable + - public String getOrderBy() + - public Integer getPageNum() + - public void setPageNum(Integer pageNum) + - public Integer getPageSize() + - public void setPageSize(Integer pageSize) + - public String getOrderByColumn() + - public void setOrderByColumn(String orderByColumn) + - public String getIsAsc() + - public void setIsAsc(String isAsc) + - public Boolean getReasonable() + - public void setReasonable(Boolean reasonable) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/page/PageParam.java +@Schema(description="分页参数") +@Data +public class PageParam extends BaseEntity implements Serializable + - private static final Integer PAGE_NO + - private static final Integer PAGE_SIZE + - public static final Integer PAGE_SIZE_NONE + - @Schema(description + - @Schema(description + - private String orderByColumn + - private String isAsc + - private String isDesc + - public String getOrderBy() + - public void setIsAsc(String isAsc) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/page/PageResult.java +@Schema(description = "分页结果") +@Data +public final class PageResult implements Serializable + - @Schema(description + - @Schema(description + - public PageResult() + - public PageResult(List list, Long total) + - public PageResult(Long total) + - public static PageResult empty() + - public static PageResult empty(Long total) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/page/SortablePageParam.java +@Schema(description = "可排序的分页参数") +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class SortablePageParam extends PageParam + - @Schema(description + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/page/SortingField.java +@Data +@NoArgsConstructor +@AllArgsConstructor +public class SortingField implements Serializable + - public static final String ORDER_ASC + - public static final String ORDER_DESC + - private String field + - private String order + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/page/TableDataInfo.java +public class TableDataInfo implements Serializable + - private static final long serialVersionUID + - private long total + - private List rows + - private int code + - private String msg + - public TableDataInfo() + - public TableDataInfo(List list, int total) + - public long getTotal() + - public void setTotal(long total) + - public List getRows() + - public void setRows(List rows) + - public int getCode() + - public void setCode(int code) + - public String getMsg() + - public void setMsg(String msg) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/page/TableSupport.java +public class TableSupport + - public static final String PAGE_NUM + - public static final String PAGE_SIZE + - public static final String ORDER_BY_COLUMN + - public static final String IS_ASC + - public static final String REASONABLE + - public static PageDomain getPageDomain() + - public static PageDomain buildPageRequest() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/redis/RedisCache.java +@SuppressWarnings(value = { "unchecked", "rawtypes" }) +@Component +public class RedisCache + - @Autowired + public RedisTemplate redisTemplate + - public void setCacheObject(final String key, final T value) + - public void setCacheObject(final String key, final T value, final Integer timeout, final TimeUnit timeUnit) + - public boolean expire(final String key, final long timeout) + - public boolean expire(final String key, final long timeout, final TimeUnit unit) + - public long getExpire(final String key) + - public Boolean hasKey(String key) + - public T getCacheObject(final String key) + - public boolean deleteObject(final String key) + - public boolean deleteObject(final Collection collection) + - public long setCacheList(final String key, final List dataList) + - public List getCacheList(final String key) + - public BoundSetOperations setCacheSet(final String key, final Set dataSet) + - public Set getCacheSet(final String key) + - public void setCacheMap(final String key, final Map dataMap) + - public Map getCacheMap(final String key) + - public void setCacheMapValue(final String key, final String hKey, final T value) + - public T getCacheMapValue(final String key, final String hKey) + - public List getMultiCacheMapValue(final String key, final Collection hKeys) + - public boolean deleteCacheMapValue(final String key, final String hKey) + - public Collection keys(final String pattern) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/text/CharsetKit.java +public class CharsetKit + - public static final String ISO_8859_1 + - public static final String UTF_8 + - public static final String GBK + - public static final Charset CHARSET_ISO_8859_1 + - public static final Charset CHARSET_UTF_8 + - public static final Charset CHARSET_GBK + - public static Charset charset(String charset) + - public static String convert(String source, String srcCharset, String destCharset) + - public static String convert(String source, Charset srcCharset, Charset destCharset) + - public static String systemCharset() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/text/Convert.java +public class Convert + - public static String toStr(Object value, String defaultValue) + - public static String toStr(Object value) + - public static Character toChar(Object value, Character defaultValue) + - public static Character toChar(Object value) + - public static Byte toByte(Object value, Byte defaultValue) + - public static Byte toByte(Object value) + - public static Short toShort(Object value, Short defaultValue) + - public static Short toShort(Object value) + - public static Number toNumber(Object value, Number defaultValue) + - public static Number toNumber(Object value) + - public static Integer toInt(Object value, Integer defaultValue) + - public static Integer toInt(Object value) + - public static Integer[] toIntArray(String str) + - public static Long[] toLongArray(String str) + - public static Integer[] toIntArray(String split, String str) + - public static Long[] toLongArray(String split, String str) + - public static String[] toStrArray(String str) + - public static String[] toStrArray(String split, String str) + - public static Long toLong(Object value, Long defaultValue) + - public static Long toLong(Object value) + - public static Double toDouble(Object value, Double defaultValue) + - public static Double toDouble(Object value) + - public static Float toFloat(Object value, Float defaultValue) + - public static Float toFloat(Object value) + - public static Boolean toBool(Object value, Boolean defaultValue) + - public static Boolean toBool(Object value) + - public static > E toEnum(Class clazz, Object value, E defaultValue) + - public static > E toEnum(Class clazz, Object value) + - public static BigInteger toBigInteger(Object value, BigInteger defaultValue) + - public static BigInteger toBigInteger(Object value) + - public static BigDecimal toBigDecimal(Object value, BigDecimal defaultValue) + - public static BigDecimal toBigDecimal(Object value) + - public static String utf8Str(Object obj) + - public static String str(Object obj, String charsetName) + - public static String str(Object obj, Charset charset) + - public static String str(byte[] bytes, String charset) + - public static String str(byte[] data, Charset charset) + - public static String str(ByteBuffer data, String charset) + - public static String str(ByteBuffer data, Charset charset) + - public static String toSBC(String input) + - public static String toSBC(String input, Set notConvertSet) + - public static String toDBC(String input) + - public static String toDBC(String text, Set notConvertSet) + - public static String digitUppercase(double n) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/core/text/StrFormatter.java +public class StrFormatter + - public static final String EMPTY_JSON + - public static final char C_BACKSLASH + - public static final char C_DELIM_START + - public static final char C_DELIM_END + - public static String format(final String strPattern, final Object... argArray) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/DataSourceFactory.java +public interface DataSourceFactory + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/DbDialect.java +public interface DbDialect + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/DbQuery.java +public interface DbQuery + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/DialectFactory.java +public class DialectFactory + - private static final DialectRegistry DIALECT_REGISTRY + - public static DbDialect getDialect(DbType dbType) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/cache/DefaultSqlCache.java +public class DefaultSqlCache extends LinkedHashMap> implements SqlCache + - private int capacity + - private long expire + - private ReentrantReadWriteLock lock + - public DefaultSqlCache(int capacity, long expire) + - @Override + public void put(String key, Object value, long ttl) + - @Override + public Object get(String key) + - @Override + public void delete(String key) + - @Override + protected boolean removeEldestEntry(Map.Entry> eldest) + - private void clean() + - static class ExpireNode + - long expire + - Object value + - public ExpireNode(long expire, Object value) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/cache/SqlCache.java +public interface SqlCache + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/constants/DbDataType.java +@Getter +public enum DbDataType + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/constants/DbQueryProperty.java +@Data +@AllArgsConstructor +public class DbQueryProperty implements Serializable + - private static final long serialVersionUID + - private String dbType + - private String host + - private String username + - private String password + - private Integer port + - private String dbName + - private String sid + - private Map config + - public DbQueryProperty(String dbType, String host, String username, String password, Integer port, String dbName, String sid) + - public void viald() + - public DbQueryProperty(String datasourceType, String ip, Long port, String datasourceConfig) + - public String trainToJdbcUrl() + - public String trainToJdbcWriterName() + - public String trainToJdbcReaderName() + - public String getDbNameTableName(String tableName) + - public String trainToJdbcWriteMode(Object columns, String writeModeType, String dbType) + - public String trainToJdbcTruncateTable(String tableName) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/constants/DbType.java +public enum DbType + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/constants/InterpretationType.java +public enum InterpretationType + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/constants/fieldtypes/DM8FieldType.java +public enum DM8FieldType + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/constants/fieldtypes/Kingbase8FieldType.java +public enum Kingbase8FieldType + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/constants/fieldtypes/MysqlFieldType.java +public enum MysqlFieldType + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/constants/fieldtypes/OracleFieldType.java +public enum OracleFieldType + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/core/DbColumn.java +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Data +public class DbColumn + - private String colName + - private String dataType + - private String dataLength + - private String dataPrecision + - private String dataScale + - private Boolean colKey + - private Boolean nullable + - private Integer colPosition + - private String dataDefault + - private String colComment + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/core/DbTable.java +@Data +public class DbTable + - private String tableName + - private String tableComment + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/core/PageResult.java +@Data +@Accessors(chain = true) +public class PageResult implements Serializable + - private static final long serialVersionUID + - private Integer pageNum + - private Integer pageSize + - private Integer total + - private List data + - public PageResult(Integer total, List data) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/datasource/AbstractDataSourceFactory.java +public abstract class AbstractDataSourceFactory implements DataSourceFactory + - @Override + public DbQuery createDbQuery(DbQueryProperty property) + - public DbQuery createDbQuery(DbQueryProperty dbQueryProperty, DataSource dataSource, DbType dbType) + - public DataSource createDataSource(DbQueryProperty property) + - protected String trainToJdbcUrl(DbQueryProperty property) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/datasource/CacheDataSourceFactoryBean.java +@Component +public class CacheDataSourceFactoryBean extends AbstractDataSourceFactory + - private static Map dataSourceMap + - @Override + public DataSource createDataSource(DbQueryProperty property) + - public static String compress(String str) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/datasource/DefaultDataSourceFactoryBean.java +@Component +@Primary // 标记为默认使用的 DataSourceFactory +public class DefaultDataSourceFactoryBean extends AbstractDataSourceFactory + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/AbstractDbDialect.java +public abstract class AbstractDbDialect implements DbDialect + - @Override + public String columns(String dbName, String tableName) + - @Override + public String tables(String dbName) + - @Override + public String tablesComment(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public String buildPaginationSql(String originalSql, long offset, long count) + - @Override + public String count(String sql) + - @Override + public String countNew(String sql, Map params) + - @Override + public Boolean validConnection(DataSource dataSource, DbQueryProperty dbQueryProperty) + - private static String buildWhereClause(Map params) + - protected String trainToJdbcUrl(DbQueryProperty property) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/DM8Dialect.java +@Slf4j +public class DM8Dialect extends AbstractDbDialect + - @Override + public String columns(String dbName, String tableName) + - @Override + public String columns(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - private List generateDmCreateSql(String tableName, String tableComment, List columns) + - private static boolean isStringTypeSwitchDEFAULT(String columnType) + - private static String mapDmColumnType(DbColumn col) + - @Override + public String tables(String dbName) + - @Override + public String tables(DbQueryProperty dbQueryProperty) + - @Override + public String buildQuerySqlFields(List columns, String tableName, DbQueryProperty dbQueryProperty) + - @Override + public String getDataStorageSize(String dbName) + - @Override + public String getDbName() + - @Override + public RowMapper columnMapper() + - @Override + public RowMapper tableMapper() + - @Override + public List validateSpecification(String tableName, String tableComment, List columns) + - @Override + public Boolean validConnection(DataSource dataSource, DbQueryProperty dbQueryProperty) + - @Override + public String getInsertOrUpdateSql(String tableName, String where, String tableFieldName, String tableFieldValue, String setValue) + - public static List validateDm8Specification(String tableName, String tableComment, List columns) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/DialectRegistry.java +public class DialectRegistry + - private final Map dialect_enum_map + - public DialectRegistry() + - public DbDialect getDialect(DbType dbType) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/KafkaDialect.java +@Slf4j +public class KafkaDialect extends AbstractDbDialect + - private static final String TEST_TOPIC + - @Override + public RowMapper tableMapper() + - @Override + public RowMapper columnMapper() + - @Override + public String columns(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - @Override + public List validateSpecification(String tableName, String tableComment, List columns) + - @Override + public String tables(DbQueryProperty dbQueryProperty) + - @Override + public String buildQuerySqlFields(List columns, String tableName, DbQueryProperty dbQueryProperty) + - @Override + public String getDataStorageSize(String dbName) + - @Override + public String getDbName() + - @Override + public Boolean validConnection(DataSource dataSource, DbQueryProperty dbQueryProperty) + - @Override + public String getInsertOrUpdateSql(String tableName, String where, String tableFieldName, String tableFieldValue, String setValue) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/Kingbase8Dialect.java +public class Kingbase8Dialect extends AbstractDbDialect + - @Override + public String columns(String dbName, String tableName) + - @Override + public String columns(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - @Override + public List validateSpecification(String tableName, String tableComment, List columns) + - private List generateKingbaseCreateSql(String tableName, String tableComment, List columns) + - private static boolean isStringTypeSwitchDEFAULT(String columnType) + - private static String mapKingbaseColumnType(DbColumn col) + - @Override + public String tables(String dbName) + - @Override + public String tables(DbQueryProperty dbQueryProperty) + - @Override + public String buildQuerySqlFields(List columns, String tableName, DbQueryProperty dbQueryProperty) + - @Override + public String buildPaginationSql(String originalSql, long offset, long count) + - @Override + public String getDataStorageSize(String dbName) + - @Override + public String getDbName() + - @Override + public String getInsertOrUpdateSql(String tableName, String where, String tableFieldName, String tableFieldValue, String setValue) + - @Override + public RowMapper columnMapper() + - @Override + public RowMapper tableMapper() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/MariaDBDialect.java +public class MariaDBDialect extends MySqlDialect + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/MySqlDialect.java +public class MySqlDialect extends AbstractDbDialect + - private static final String[] MYSQL_RESERVED_WORDS + - @Override + public RowMapper columnMapper() + - @Override + public String columns(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - public static String escapeReservedKeyword(String colName) + - @Override + public List validateSpecification(String tableName, String tableComment, List columns) + - public static String generateColumnSQLMySql(String columnType, String columnLength, String columnScale, int maxLength, int maxScale) + - @Override + public String tables(DbQueryProperty dbQueryProperty) + - @Override + public String buildQuerySqlFields(List columns, String tableName, DbQueryProperty dbQueryProperty) + - @Override + public String getDataStorageSize(String dbName) + - @Override + public String getDbName() + - @Override + public String getInsertOrUpdateSql(String tableName, String where, String tableFieldName, String tableFieldValue, String setValue) + - @Override + public RowMapper tableMapper() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/Oracle12cDialect.java +public class Oracle12cDialect extends OracleDialect + - @Override + public String buildPaginationSql(String originalSql, long offset, long count) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - private List generateOracleCreateSql(String tableName, String tableComment, List columns) + - private static boolean isStringTypeSwitchDEFAULT(String columnType) + - private static boolean isStringTypeSwitchNullableFlag(String columnType) + - private static String mapOracleColumnType(DbColumn col) + - public static String generateColumnDefinitionOracle(Long columnLength, long maxLength, boolean includeScale, Long columnScale) + - @Override + public String buildQuerySqlFields(List columns, String tableName, DbQueryProperty dbQueryProperty) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/OracleDialect.java +public class OracleDialect extends AbstractDbDialect + - @Override + public String columns(String dbName, String tableName) + - @Override + public String columns(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - @Override + public List validateSpecification(String tableName, String tableComment, List columns) + - private List generateOracleCreateSql(String tableName, String tableComment, List columns) + - private static boolean isStringTypeSwitchDEFAULT(String columnType) + - private static boolean isStringTypeSwitchNullableFlag(String columnType) + - private static String mapOracleColumnType(DbColumn col) + - public static String generateColumnDefinitionOracle(Long columnLength, long maxLength, boolean includeScale, Long columnScale) + - @Override + public String tables(String dbName) + - @Override + public String tables(DbQueryProperty dbQueryProperty) + - @Override + public String buildQuerySqlFields(List columns, String tableName, DbQueryProperty dbQueryProperty) + - @Override + public String buildPaginationSql(String sql, long offset, long count) + - @Override + public RowMapper columnLongMapper() + - @Override + public String getDataStorageSize(String dbName) + - @Override + public String getDbName() + - @Override + public String getInsertOrUpdateSql(String tableName, String where, String tableFieldName, String tableFieldValue, String setValue) + - @Override + public RowMapper columnMapper() + - @Override + public RowMapper tableMapper() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/PostgreDialect.java +public class PostgreDialect extends AbstractDbDialect + - @Override + public String columns(String dbName, String tableName) + - @Override + public String columns(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - @Override + public List validateSpecification(String tableName, String tableComment, List columns) + - @Override + public String tables(String dbName) + - @Override + public String tables(DbQueryProperty dbQueryProperty) + - @Override + public String buildQuerySqlFields(List columns, String tableName, DbQueryProperty dbQueryProperty) + - @Override + public String buildPaginationSql(String originalSql, long offset, long count) + - @Override + public String getDataStorageSize(String dbName) + - @Override + public String getDbName() + - @Override + public String getInsertOrUpdateSql(String tableName, String where, String tableFieldName, String tableFieldValue, String setValue) + - @Override + public RowMapper columnMapper() + - @Override + public RowMapper tableMapper() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/SQLServer2008Dialect.java +public class SQLServer2008Dialect extends AbstractDbDialect + - @Override + public String columns(String dbName, String tableName) + - @Override + public String columns(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - @Override + public List validateSpecification(String tableName, String tableComment, List columns) + - @Override + public String tables(String dbName) + - @Override + public String tables(DbQueryProperty dbQueryProperty) + - @Override + public String buildQuerySqlFields(List columns, String tableName, DbQueryProperty dbQueryProperty) + - private static String getOrderByPart(String sql) + - @Override + public String buildPaginationSql(String originalSql, long offset, long size) + - @Override + public String getDataStorageSize(String dbName) + - @Override + public String getDbName() + - @Override + public String getInsertOrUpdateSql(String tableName, String where, String tableFieldName, String tableFieldValue, String setValue) + - @Override + public RowMapper columnMapper() + - @Override + public RowMapper tableMapper() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/SQLServerDialect.java +public class SQLServerDialect extends SQLServer2008Dialect + - @Override + public String buildPaginationSql(String originalSql, long offset, long size) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/dialect/UnknownDialect.java +public class UnknownDialect extends AbstractDbDialect + - @Override + public String columns(String dbName, String tableName) + - @Override + public String columns(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public String generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List someInternalSqlGenerator(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - @Override + public List validateSpecification(String tableName, String tableComment, List columns) + - @Override + public String tables(String dbName) + - @Override + public String tables(DbQueryProperty dbQueryProperty) + - @Override + public String buildQuerySqlFields(List columns, String tableName, DbQueryProperty dbQueryProperty) + - @Override + public String buildPaginationSql(String sql, long offset, long count) + - @Override + public String count(String sql) + - @Override + public String countNew(String sql, Map params) + - @Override + public Boolean validConnection(DataSource dataSource, DbQueryProperty dbQueryProperty) + - @Override + public String getInsertOrUpdateSql(String tableName, String where, String tableFieldName, String tableFieldValue, String setValue) + - @Override + public RowMapper columnMapper() + - @Override + public RowMapper tableMapper() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/exception/DataQueryException.java +public class DataQueryException extends RuntimeException + - public DataQueryException(String message) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/query/AbstractDbQueryFactory.java +@Setter +public abstract class AbstractDbQueryFactory implements DbQuery + - protected DataSource dataSource + - protected JdbcTemplate jdbcTemplate + - protected DbDialect dbDialect + - private DbQueryProperty dbQueryProperty + - @Override + public Connection getConnection() + - @Override + public boolean valid() + - @Override + public void close() + - @Override + public List getTableColumns(String dbName, String tableName) + - @Override + public List getTableColumns(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public int generateCheckTableExistsSQL(DbQueryProperty dbQueryProperty, String tableName) + - @Override + public List generateCreateTableSQL(DbQueryProperty dbQueryProperty, String tableName, String tableComment, List dbColumnList) + - @Override + public List getTables(String dbName) + - @Override + public List getTables(DbQueryProperty dbQueryProperty) + - @Override + public int count(String sql) + - @Override + public int count(String sql, Object[] args) + - @Override + public int count(String sql, Map params) + - @Override + public int countNew(String sql, Map params) + - @Override + public List> queryList(String sql) + - @Override + public List> queryDbColumnByList(List columns, String tableName, DbQueryProperty dbQueryProperty, long offset, long size) + - @Override + public List> queryList(String sql, Object[] args) + - @Override + public PageResult> queryByPage(String sql, long offset, long size) + - @Override + public PageResult> queryByPage(String sql, Object[] args, long offset, long size) + - @Override + public PageResult> queryByPage(String sql, Map params, long offset, long size, Integer cache) + - @Override + public List> queryList(String sql, Map params, Integer cache) + - @Override + public Map queryOne(String sql, Map params, Integer cache) + - @Override + public int update(String sql) + - @Override + public void execute(String sql) + - @Override + public int[] batchUpdate(String sql) + - @Override + public int isTableExists(String sql) + - protected String trainToJdbcUrl(DbQueryProperty property) + - @Override + public Integer getDataStorageSize() + - @Override + public Boolean copyTable(Connection conn, DbQueryProperty dbQueryProperty, String tableName, String newTableName) + - @Override + public String getInsertOrUpdateSql(DbQueryProperty writerProperty, String tableName, List selectedColumns, List column) + - public static String onDuplicateKeyUpdateString(List columnHolders) + - public static String onMergeIntoDoString(List selectedColumns, List column) + - @Override + public List getColumnsByQuerySql(String querySql) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/query/CacheDbQueryFactoryBean.java +public class CacheDbQueryFactoryBean extends AbstractDbQueryFactory + - private static long DEFAULT_EXPIRE + - private static DefaultSqlCache sqlCache + - private T putCacheValue(String key, T value, long ttl) + - @Override + public List getTableColumns(String dbName, String tableName) + - @Override + public List> queryList(String sql) + - @Override + public List> queryList(String sql, Object[] args) + - @Override + public List> queryList(String sql, Map params, Integer cache) + - @Override + public Map queryOne(String sql, Map params, Integer cache) + - @Override + public PageResult> queryByPage(String sql, long offset, long size) + - @Override + public PageResult> queryByPage(String sql, Object[] args, long offset, long size) + - @Override + public PageResult> queryByPage(String sql, Map params, long offset, long size, Integer cache) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/query/DefaultDbQueryFactoryBean.java +public class DefaultDbQueryFactoryBean extends AbstractDbQueryFactory + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/utils/MD5Util.java +public class MD5Util + - private final byte[] DESIV + - private final String deSkey + - private AlgorithmParameterSpec iv + - private Key key + - private String charset + - private static volatile MD5Util instance + - private MD5Util() throws Exception + - public static MD5Util getInstance() throws Exception + - public static void main(String[] args) + - public String encode(String data) throws Exception + - public String decode(String data) throws Exception + - private static String getMD5Value(String sourceStr) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/utils/ConversionUtils.java +public class ConversionUtils + - public static Long getStringToLong(String dataLength) + - public static Integer getStringToInt(String dataLength) + - public static Double getStringToDouble(String dataLength) + - public static Boolean getStringToBoolean(String value) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/utils/StringUtils.java +public class StringUtils extends org.apache.commons.lang3.StringUtils + - private static final String NULLSTR + - private static final char SEPARATOR + - private static final char ASTERISK + - public static T nvl(T value, T defaultValue) + - public static boolean isEmpty(Collection coll) + - public static boolean isNotEmpty(Collection coll) + - public static boolean isEmpty(Object[] objects) + - public static boolean isNotEmpty(Object[] objects) + - public static boolean isEmpty(Map map) + - public static boolean isNotEmpty(Map map) + - public static boolean isEmpty(String str) + - public static boolean isNotEmpty(String str) + - public static boolean isNull(Object object) + - public static boolean isNotNull(Object object) + - public static boolean isArray(Object object) + - public static String trim(String str) + - public static String hide(CharSequence str, int startInclude, int endExclude) + - public static String substring(final String str, int start) + - public static String substring(final String str, int start, int end) + - public static boolean hasText(String str) + - private static boolean containsText(CharSequence str) + - public static String format(String template, Object... params) + - public static boolean ishttp(String link) + - public static final Set str2Set(String str, String sep) + - public static final List str2List(String str, String sep, boolean filterBlank, boolean trim) + - public static boolean containsAny(Collection collection, String... array) + - public static boolean containsAnyIgnoreCase(CharSequence cs, CharSequence... searchCharSequences) + - public static String toUnderScoreCase(String str) + - public static boolean inStringIgnoreCase(String str, String... strs) + - public static String convertToCamelCase(String name) + - public static String toCamelCase(String s) + - public static boolean matches(String str, List strs) + - public static boolean isMatch(String pattern, String url) + - @SuppressWarnings("unchecked") + public static T cast(Object obj) + - public static final String padl(final Number num, final int size) + - public static final String padl(final String s, final int size, final char c) + - private static final String ALPHANUMERIC + - public static String generateRandomString() + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/utils/BeanUtils.java +public class BeanUtils extends org.springframework.beans.BeanUtils + - private static final int BEAN_METHOD_PROP_INDEX + - private static final Pattern GET_PATTERN + - private static final Pattern SET_PATTERN + - public static void copyBeanProp(Object dest, Object src) + - public static List getSetterMethods(Object obj) + - public static List getGetterMethods(Object obj) + - public static boolean isMethodPropEquals(String m1, String m2) + +### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/utils/BeanValidators.java +public class BeanValidators + - public static void validateWithException(Validator validator, Object object, Class... groups) + throws ConstraintViolationException -| 优势点 | 描述 | -|------------------------|----------------------------------------------------------------------------------------| -| 🧩 **统一标准** | 建立统一的数据规范、模型和指标体系，消除“数据孤岛”，提升全局协同效率。 | -| ⚙️ **敏捷支撑** | 快速响应业务变化，按需提供数据服务和分析支持，加速业务创新。 | -| 🔒 **安全可控** | 数据权限精细化管理，保障数据合规使用，支撑审计与监管要求。 | - -## ✅ 已有功能一览 - -| 功能模块 | 功能描述 | 状态 | -|--------|-----------------------------------------------------------------------------------------------------------------------------------|--------| -| 系统管理 | 包含系统配置、用户与权限管理、运行监控、日志管理、服务器资源监控和工作空间管理，支持将任务拆分给不同项目组协作，适应前店后厂的工作流程。 | ✅ 完成 | -| 数据标准管理 | 管理逻辑模型（表级，支持关联数据元）和数据元模型（字段级，绑定规范、稽查与清洗），实现结构化数据规范和质量管控。 | ✅ 完成 | -| 数据集成（ETL） | 支持数据库、文件、消息队列等多种数据源接入，提供可视化ETL设计、多任务调度与监控，支持数据清洗规则应用，包含运行日志和执行配置管理。 | ✅ 完成 | -| 元数据管理 | 集中管理数据表、字段、稽查、清洗指标等元信息，建立统一的数据资产目录，支撑数据查找、血缘分析和结构维护。 | ✅ 完成 | -| 数据质量管理 | 支持基于预置的稽查规则及数据清洗规则，实现对数据准确性、一致性和完整性的自动校验与修正。 | ✅ 完成 | -| 数据资产管理 | 通过目录结构对数据资产进行分级分类管理，支持全生命周期跟踪与维护。 | ✅ 完成 | -| 统一标签管理 | 支持标签自动生成与多维度标签筛选，便于数据分类和精准检索。 | ✅ 完成 | -| 数据开发 | 支持达梦8、MySQL、Oracle、人大金仓等主流数据库任务开发，包含调度配置和日志管理，基于数据仓库，通过SQL及大数据脚本结合业务需求开发和加工业务数据。 | ✅ 完成 | -| 数据查询 | 提供灵活且高效的交互式查询工具，支持快速分析和挖掘结构化数据。 | ✅ 完成 | -| 数据服务管理 | 将数据资产封装为 API 服务，支持外部 API 接口转发，包含 API 日志、数据脱敏、数据接口和服务接口管理及调用日志监控。 | ✅ 完成    | - -## 🚧 未来开发计划 - -| 功能模块 | 功能描述 | 状态 | -|------------------------|----------------------------------------------------------------|----------| -| 作业调度中心 | 实现任务编排、任务依赖管理、定时执行、失败重试、日志追踪等调度能力，提升任务执行的自动化与可控性。 | 🔨 开发中 | -| 数据集成（增强） | 增强异构数据源、文件、接口、网络爬虫的采集与同步能力，支持更多类型转换与接入能力。 | 🔨 开发中 | -| 离线数据开发 | 支持基于 Spark、Hive、SQL 等任务类型的离线数据处理，适用于大批量数据的周期性加工与分析，规划统一的作业管理与执行入口。 | 🔨 开发中 | -| 实时数据开发 | 支持基于 Flink、Kafka 等流处理引擎的实时任务开发，实现低延迟数据处理与指标计算，适用于实时监控与事件驱动业务场景。 | ⏳ 规划中 | -| 数据质量管理（增强） | 基于规则引擎实现数据质量检测，支持质量报告生成及质量指标的监控与统计，辅助识别和改进数据中的准确性与完整性问题。 | ⏳ 规划中 | -| 数据比对管理 | 支持数据库连接配置及任务调度，自动执行结构和内容比对任务，生成详细日志，便于跨源数据一致性验证。 | ⏳ 规划中 | -| 元数据管理（扩展） | 增强数据字段、数据地图、SQL 控制台、变更记录等元数据功能，逐步实现血缘、影响分析与结构可视化。 | ⏳ 规划中 | -| 数据血缘分析 | 支持字段级血缘追踪与可视化图谱展示，帮助用户清晰理解数据流向与依赖关系，提升数据透明度与可追溯性。 | ⏳ 规划中 | -| 数据图谱管理 | 基于数据仓库构建业务实体、事实表与维度表之间的图谱关系，支持多层模型梳理与可视化展示，增强对数据逻辑结构和业务依赖的理解与维护能力。 | ⏳ 规划中 | -| 数据可视化 | 支持 BI 报表、数据看板、大屏和多种数据图表展示，支持拖拽配置、权限控制与动态联动。 | ⏳ 规划中 | -| AI 辅助开发与治理 | 引入 AI 技术，重点支持离线与实时数据开发中的SQL智能补全、自动优化、异常检测及调试辅助，提升开发效率和数据质量。 | ⏳ 规划中    | - -💡 如您有好的建议或功能需求，欢迎 [提交Issue](https://gitee.com/qiantongtech/qData/issues)，与我们共同完善数据中台功能。 - -## 🚨 默认端口 -| 端口号 | 协议 | 说明 | -|-------| ----- | -------------------------------- | -| 8080 | http | 平台前端默认端口 | -| 8081 | http | 平台后端默认端口 | -| 2181 | tcp | ZooKeeper 默认客户端连接端口 | -| 8020 | tcp | Hadoop HDFS NameNode RPC 端口 | -| 9870 | http | Hadoop HDFS NameNode Web UI端口 | -| 9000 | tcp | Hadoop HDFS 默认FS端口（部分配置）| -| 10000 | tcp | HiveServer2 默认端口 | -| 9083 | tcp | Hive Metastore Thrift 服务端口 | -| 16010 | http | HBase Master Web UI 端口 | -| 60010 | http | HBase RegionServer Web UI 端口 | -| 7077 | tcp | Spark Master 端口 | -| 8080 | http | Spark Master Web UI 端口 | -| 4040 | http | Spark 应用程序 Web UI 端口 | -| 18080 | http | Spark History Server Web UI 端口 | - -**访问地址**：127.0.0.1:81，账号：qData 密码：qData123 - -[//]: # (## 🧩 架构图) - -[//]: # (![framework.png](images%2Fframework.png)) - -## 🛠️ 技术栈 -qData 平台采用前后端分离架构，后端基于 Spring Boot，前端基于 Vue 3，整合常用中间件与数据工具，构建一站式数据中台解决方案。 - -### 🖥️ 后端技术栈 - -| 分类 | 技术 | 描述 | -|----------------|----------------------------------|----------------------------| -| 服务框架 | Spring Boot | 提供快速开发能力 | -| 权限控制 | Spring Security | 实现用户权限认证与控制 | -| 数据库 | MySQL、PostgreSQL、达梦8、人大金仓 | 持久化存储与配置管理 | -| ORM 框架 | MyBatis-Plus | 简化数据库操作 | -| 缓存与并发控制 | Redis | 支持缓存、分布式锁等 | -| 消息队列 | RabbitMQ | 实现异步通信与解耦处理 | - -### 🌐 前端技术栈 - -| 分类 | 技术 | 描述 | -|----------|-------------|------------------| -| 框架 | Vue 3 | 现代化响应式框架 | -| 组件库 | Element UI | 常用 UI 组件支持 | -| 构建工具 | Vite | 快速开发与构建工具 | - - -### 🧩 第三方依赖 - -| 分类 | 组件 / 工具 | 描述 | -|----------------------|------------------|----------------------------------------------------| -| 数据仓库 | Hive | 支持数据建模、分区管理及元数据维护 | -| 计算引擎与数据集成 | Spark | 批流一体，支持 ETL 数据处理 | -| 大数据存储 | Hive、HBase | 支持海量非结构化与半结构化数据存储 | -| 作业调度与任务管理 | DolphinScheduler | 提供可视化任务编排、依赖管理及调度能力 | - - -## 🏗️ 部署要求 -在部署 qData 之前，请确保以下环境和工具已正确安装： - -### 🖥️ 服务端环境要求（后端） - -| 项目 | 推荐版本 | 说明 | -|---------------|--------------|---------------------| -| JDK | 1.8 或以上 | 建议使用 OpenJDK 8 或 11 | -| Maven | 3.6+ | 项目构建与依赖管理 | -| 达梦8 | 8.0 | 关系型数据库（可切至MySQL） | -| Redis | 5.0+ | 缓存与消息功能支持 | -| RabbitMQ | 可选 |用于任务调度、异步通信等功能。 | -| 操作系统 | Windows / Linux / Mac | 通用环境均可运行 | - -### 🌐 前端环境要求（可选，若需本地构建） - -| 项目 | 推荐版本 | 说明 | -|------------------|------|------------------------| -| Node.js | 16+ | 构建工具依赖 | -| npm | 10+ | 包管理器| -| Vite | 最新版 | 脚手架工具 | - -## 🚨 商用授权 -> **版权申明**： -> - 产品的迭代更新需要大家支持，在第三方获取到的商业版本源码，没有可靠性且未经授权，由此带来的法律后果自行承担。 -> - 授权用户不能直接销售和分发 **千数平台（qData）** 商业版源码。 -> - 授权用户二次开发和修改部分的源码可以申请软件著作权，不可以申请 **千数平台（qData）** 源码部分的软件著作权。 - -| 系统版本 | 开源版本 | 商业版本 | -|------|-------------------------------|--------------------------------------------------| -| 许可协议 | 遵循Apache License 2.0协议（可商用，不可去除 Logo 和版权） | 永久商业授权（包含源码、文档及品牌使用授权），购买后提供发票/合同和授权书 | -| 系统用途 | 商业项目可用，适合学习、评估、初步部署等轻量级使用场景 | 适用于政企项目、定制化开发、正式生产环境部署等中大型场景 | -| 系统功能 | 数据标准、数据集成、数据资产、数据开发、系统管理等 | 开源版所有功能、数据集成（增强）、实时数据开发、数据血缘分析、数据可视化、AI 辅助开发与治理等 | -| 授权费用 | 免费，可商用（如需修改 Logo 和版权，品牌授权费3万元） | 20万元 | -| 相关服务 | 定期更新 | 加入商业版私有仓库，赠送一年的订阅更新 | -| 技术支持 | QQ群交流 | 企业用户群交流，提供必要技术支持 | -| 代码更新 | Gitee、Github更新 | Git私服更新 | - -## 🚀 快速开始 -👉 点击下方按钮，查看完整的安装与部署指南： - -🧭 查看快速开始文档 - -## 👥 QQ交流群 -欢迎加入 qData 官方 QQ 交流群，获取最新动态、技术支持与使用交流。 - -[![加入QQ群](https://img.shields.io/badge/QQ群-814145903-blue.svg)](https://qm.qq.com/cgi-bin/qm/qr?k=fOuDDH-dLk4lhYezFO_Vg1IwSD58SxP2&jump_from=webapi&authKey=eCpXabAvZN5rc6jK/ukudn7G0mhMLQOjD7ov03xAfO5OqHNI7uvzujMqdXUwmNlK) - -## 🖼️ 系统配图 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
登录页首页
稽查规则清洗规则
逻辑模型数据元
映射数据连接
数据发现资产地图
数据集成任务可视化ETL
数据服务执行日志
发布服务日志
+### qdata-framework/qdata-common/src/main/java/tech/qiantong/qdata/common/database/utils/CollectionUtils.java +public class CollectionUtils + - public static boolean containsAny(Object source, Object... targets) + - public static boolean isAnyEmpty(Collection... collections) + - public static boolean anyMatch(Collection from, Predicate predicate) + - public static List filterList(Collection from, Predicate predicate) + - public static List distinct(Collection from, Function keyMapper) + - public static List distinct(Collection from, Function keyMapper, BinaryOperator cover) + - public static List convertList(T[] from, Function func) + - public static List convertList(Collection from, Function func) + - public static List convertList(Collection from, Function func, Predicate filter) + - public static List convertListByFlatMap(Collection from, + Function> func) + - public static List convertListByFlatMap(Collection from, + Function mapper, + Function> func) + - public static List mergeValuesFromMap(Map> map) + - public static Set convertSet(Collection from) + - public static Set convertSet(Collection from, Function func) + - public static Set convertSet(Collection from, Function func, Predicate filter) + - public static Map convertMapByFilter(Collection from, Predicate filter, Function keyFunc) + - public static Set convertSetByFlatMap(Collection from, + Function> func) + - public static Set convertSetByFlatMap(Collection from, + Function mapper, + Function> func) + - public static Map convertMap(Collection from, Function keyFunc) + - public static Map convertMap(Collection from, Function keyFunc, Supplier> supplier) + - public static Map convertMap(Collection from, Function keyFunc, Function valueFunc) + - public static Map convertMap(Collection from, Function keyFunc, Function valueFunc, BinaryOperator mergeFunction) + - public static Map convertMap(Collection from, Function keyFunc, Function valueFunc, BinaryOperator mergeFunction, Supplier> supplier) + - public static Map> convertMultiMap(Collection from, Function keyFunc) + - public static Map> convertMultiMap(Collection from, Function keyFunc, Function valueFunc) + - public static Map> convertMultiMap2(Collection from, Function keyFunc, Function valueFunc) + - public static Map convertImmutableMap(Collection from, Function keyFunc) + - public static List> diffList(Collection oldList, Collection newList, + BiFunction sameFunc) + - public static T getFirst(List from) + - public static T findFirst(Collection from, Predicate predicate) + - public static > V getMaxValue(Collection from, Function valueFunc \ No newline at end of file -- Gitee